New Token data member for splitting token value when we need 2 values.

This commit is contained in:
Ferdinand Majerech 2014-07-25 03:33:08 +02:00
parent 2228e92a43
commit e88842573f

View file

@ -52,18 +52,27 @@ struct Token
{
@disable int opCmp(ref Token);
// 16B
/// Value of the token, if any.
string value;
// 4B
/// Start position of the token in file/stream.
Mark startMark;
// 4B
/// End position of the token in file/stream.
Mark endMark;
// 1B
/// Token type.
TokenID id;
// 1B
/// Style of scalar token, if this is a scalar token.
ScalarStyle style;
// 1B
/// Encoding, if this is a stream start token.
Encoding encoding;
// 4B
/// Used to split value into 2 substrings for tokens that need 2 values (tagToken)
uint valueDivider;
/// Get string representation of the token ID.
@property string idString() @safe pure const {return id.to!string;}