dyaml/source/dyaml/scanner.d
2013-04-24 01:32:16 +02:00

1652 lines
60 KiB
D

// Copyright Ferdinand Majerech 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
/**
* YAML scanner.
* Code based on PyYAML: http://www.pyyaml.org
*/
module dyaml.scanner;
import core.stdc.string;
import std.algorithm;
import std.array;
import std.container;
import std.conv;
import std.ascii : isAlphaNum, isDigit, isHexDigit;
import std.exception;
import std.string;
import std.typecons;
import std.utf;
import dyaml.fastcharsearch;
import dyaml.escapes;
import dyaml.exception;
import dyaml.queue;
import dyaml.reader;
import dyaml.style;
import dyaml.token;
package:
/**
* Scanner produces tokens of the following types:
* STREAM-START
* STREAM-END
* DIRECTIVE(name, value)
* DOCUMENT-START
* DOCUMENT-END
* BLOCK-SEQUENCE-START
* BLOCK-MAPPING-START
* BLOCK-END
* FLOW-SEQUENCE-START
* FLOW-MAPPING-START
* FLOW-SEQUENCE-END
* FLOW-MAPPING-END
* BLOCK-ENTRY
* FLOW-ENTRY
* KEY
* VALUE
* ALIAS(value)
* ANCHOR(value)
* TAG(value)
* SCALAR(value, plain, style)
*/
/**
* Marked exception thrown at scanner errors.
*
* See_Also: MarkedYAMLException
*/
class ScannerException : MarkedYAMLException
{
mixin MarkedExceptionCtors;
}
private alias ScannerException Error;
///Generates tokens from data provided by a Reader.
final class Scanner
{
private:
/**
* A simple key is a key that is not denoted by the '?' indicator.
* For example:
* ---
* block simple key: value
* ? not a simple key:
* : { flow simple key: value }
* We emit the KEY token before all keys, so when we find a potential
* simple key, we try to locate the corresponding ':' indicator.
* Simple keys should be limited to a single line and 1024 characters.
*
* 16 bytes on 64-bit.
*/
static struct SimpleKey
{
///Character index in reader where the key starts.
uint charIndex = uint.max;
///Index of the key token from start (first token scanned being 0).
uint tokenIndex;
///Line the key starts at.
uint line;
///Column the key starts at.
ushort column;
///Is this required to be a simple key?
bool required;
///Is this struct "null" (invalid)?.
bool isNull;
}
///Block chomping types.
enum Chomping
{
///Strip all trailing line breaks. '-' indicator.
Strip,
///Line break of the last line is preserved, others discarded. Default.
Clip,
///All trailing line breaks are preserved. '+' indicator.
Keep
}
///Reader used to read from a file/stream.
Reader reader_;
///Are we done scanning?
bool done_;
///Level of nesting in flow context. If 0, we're in block context.
uint flowLevel_;
///Current indentation level.
int indent_ = -1;
///Past indentation levels. Used as a stack.
Array!int indents_;
///Processed tokens not yet emitted. Used as a queue.
Queue!Token tokens_;
///Number of tokens emitted through the getToken method.
uint tokensTaken_;
/**
* Can a simple key start at the current position? A simple key may
* start:
* - at the beginning of the line, not counting indentation spaces
* (in block context),
* - after '{', '[', ',' (in the flow context),
* - after '?', ':', '-' (in the block context).
* In the block context, this flag also signifies if a block collection
* may start at the current position.
*/
bool allowSimpleKey_ = true;
///Possible simple keys indexed by flow levels.
SimpleKey[] possibleSimpleKeys_;
///Used for constructing strings while limiting reallocation.
Appender!(dchar[]) appender_;
public:
///Construct a Scanner using specified Reader.
this(Reader reader) @trusted
{
//Return the next token, but do not delete it from the queue
reader_ = reader;
appender_ = appender!(dchar[])();
fetchStreamStart();
}
///Destroy the scanner.
@trusted ~this()
{
clear(tokens_);
clear(indents_);
clear(possibleSimpleKeys_);
possibleSimpleKeys_ = null;
clear(appender_);
reader_ = null;
}
/**
* Check if the next token is one of specified types.
*
* If no types are specified, checks if any tokens are left.
*
* Params: ids = Token IDs to check for.
*
* Returns: true if the next token is one of specified types,
* or if there are any tokens left if no types specified.
* false otherwise.
*/
bool checkToken(const TokenID[] ids ...) @safe
{
//Check if the next token is one of specified types.
while(needMoreTokens()){fetchToken();}
if(!tokens_.empty)
{
if(ids.length == 0){return true;}
else
{
const nextId = tokens_.peek().id;
foreach(id; ids)
{
if(nextId == id){return true;}
}
}
}
return false;
}
/**
* Return the next token, but keep it in the queue.
*
* Must not be called if there are no tokens left.
*/
ref const(Token) peekToken() @safe
{
while(needMoreTokens){fetchToken();}
if(!tokens_.empty){return tokens_.peek();}
assert(false, "No token left to peek");
}
/**
* Return the next token, removing it from the queue.
*
* Must not be called if there are no tokens left.
*/
Token getToken() @safe
{
while(needMoreTokens){fetchToken();}
if(!tokens_.empty)
{
++tokensTaken_;
return tokens_.pop();
}
assert(false, "No token left to get");
}
private:
///Determine whether or not we need to fetch more tokens before peeking/getting a token.
@property bool needMoreTokens() pure @safe
{
if(done_) {return false;}
if(tokens_.empty){return true;}
///The current token may be a potential simple key, so we need to look further.
stalePossibleSimpleKeys();
return nextPossibleSimpleKey() == tokensTaken_;
}
///Fetch at token, adding it to tokens_.
void fetchToken() @trusted
{
///Eat whitespaces and comments until we reach the next token.
scanToNextToken();
//Remove obsolete possible simple keys.
stalePossibleSimpleKeys();
//Compare current indentation and column. It may add some tokens
//and decrease the current indentation level.
unwindIndent(reader_.column);
//Get the next character.
const dchar c = reader_.peek();
//Fetch the token.
if(c == '\0') {return fetchStreamEnd();}
if(checkDirective()) {return fetchDirective();}
if(checkDocumentStart()) {return fetchDocumentStart();}
if(checkDocumentEnd()) {return fetchDocumentEnd();}
//Order of the following checks is NOT significant.
if(c == '[') {return fetchFlowSequenceStart();}
if(c == '{') {return fetchFlowMappingStart();}
if(c == ']') {return fetchFlowSequenceEnd();}
if(c == '}') {return fetchFlowMappingEnd();}
if(c == ',') {return fetchFlowEntry();}
if(checkBlockEntry()) {return fetchBlockEntry();}
if(checkKey()) {return fetchKey();}
if(checkValue()) {return fetchValue();}
if(c == '*') {return fetchAlias();}
if(c == '&') {return fetchAnchor();}
if(c == '!') {return fetchTag();}
if(c == '|' && flowLevel_ == 0){return fetchLiteral();}
if(c == '>' && flowLevel_ == 0){return fetchFolded();}
if(c == '\'') {return fetchSingle();}
if(c == '\"') {return fetchDouble();}
if(checkPlain()) {return fetchPlain();}
throw new Error(format("While scanning for the next token, found "
"character \'%s\', index %s that cannot start any token"
, c, to!int(c)), reader_.mark);
}
///Return the token number of the nearest possible simple key.
uint nextPossibleSimpleKey() pure @safe nothrow
{
uint minTokenNumber = uint.max;
foreach(k, ref simpleKey; possibleSimpleKeys_)
{
if(simpleKey.isNull){continue;}
minTokenNumber = min(minTokenNumber, simpleKey.tokenIndex);
}
return minTokenNumber;
}
/**
* Remove entries that are no longer possible simple keys.
*
* According to the YAML specification, simple keys
* - should be limited to a single line,
* - should be no longer than 1024 characters.
* Disabling this will allow simple keys of any length and
* height (may cause problems if indentation is broken though).
*/
void stalePossibleSimpleKeys() pure @safe
{
foreach(level, ref key; possibleSimpleKeys_)
{
if(key.isNull){continue;}
if(key.line != reader_.line || reader_.charIndex - key.charIndex > 1024)
{
enforce(!key.required,
new Error("While scanning a simple key",
Mark(key.line, key.column),
"could not find expected ':'", reader_.mark));
key.isNull = true;
}
}
}
/**
* Check if the next token starts a possible simple key and if so, save its position.
*
* This function is called for ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
*/
void savePossibleSimpleKey() pure @system
{
//Check if a simple key is required at the current position.
const required = (flowLevel_ == 0 && indent_ == reader_.column);
assert(allowSimpleKey_ || !required, "A simple key is required only if it is "
"the first token in the current line. Therefore it is always allowed.");
if(!allowSimpleKey_){return;}
//The next token might be a simple key, so save its number and position.
removePossibleSimpleKey();
const tokenCount = tokensTaken_ + cast(uint)tokens_.length;
const line = reader_.line;
const column = reader_.column;
const key = SimpleKey(cast(uint)reader_.charIndex,
tokenCount,
line,
column < ushort.max ? cast(ushort)column : ushort.max,
required);
if(possibleSimpleKeys_.length <= flowLevel_)
{
const oldLength = possibleSimpleKeys_.length;
possibleSimpleKeys_.length = flowLevel_ + 1;
//No need to initialize the last element, it's already done in the next line.
possibleSimpleKeys_[oldLength .. flowLevel_] = SimpleKey.init;
}
possibleSimpleKeys_[flowLevel_] = key;
}
///Remove the saved possible key position at the current flow level.
void removePossibleSimpleKey() pure @safe
{
if(possibleSimpleKeys_.length <= flowLevel_){return;}
if(!possibleSimpleKeys_[flowLevel_].isNull)
{
const key = possibleSimpleKeys_[flowLevel_];
enforce(!key.required,
new Error("While scanning a simple key", Mark(key.line, key.column),
"could not find expected ':'", reader_.mark));
possibleSimpleKeys_[flowLevel_].isNull = true;
}
}
/**
* Decrease indentation, removing entries in indents_.
*
* Params: column = Current column in the file/stream.
*/
void unwindIndent(const int column) @trusted
{
if(flowLevel_ > 0)
{
//In flow context, tokens should respect indentation.
//The condition should be `indent >= column` according to the spec.
//But this condition will prohibit intuitively correct
//constructions such as
//key : {
//}
//In the flow context, indentation is ignored. We make the scanner less
//restrictive than what the specification requires.
//if(pedantic_ && flowLevel_ > 0 && indent_ > column)
//{
// throw new Error("Invalid intendation or unclosed '[' or '{'",
// reader_.mark)
//}
return;
}
//In block context, we may need to issue the BLOCK-END tokens.
while(indent_ > column)
{
indent_ = indents_.back;
indents_.length = indents_.length - 1;
tokens_.push(blockEndToken(reader_.mark, reader_.mark));
}
}
/**
* Increase indentation if needed.
*
* Params: column = Current column in the file/stream.
*
* Returns: true if the indentation was increased, false otherwise.
*/
bool addIndent(int column) @trusted
{
if(indent_ >= column){return false;}
indents_ ~= indent_;
indent_ = column;
return true;
}
///Add STREAM-START token.
void fetchStreamStart() @safe nothrow
{
tokens_.push(streamStartToken(reader_.mark, reader_.mark, reader_.encoding));
}
///Add STREAM-END token.
void fetchStreamEnd() @safe
{
//Set intendation to -1 .
unwindIndent(-1);
removePossibleSimpleKey();
allowSimpleKey_ = false;
clear(possibleSimpleKeys_);
tokens_.push(streamEndToken(reader_.mark, reader_.mark));
done_ = true;
}
///Add DIRECTIVE token.
void fetchDirective() @safe
{
//Set intendation to -1 .
unwindIndent(-1);
//Reset simple keys.
removePossibleSimpleKey();
allowSimpleKey_ = false;
tokens_.push(scanDirective());
}
///Add DOCUMENT-START or DOCUMENT-END token.
void fetchDocumentIndicator(TokenID id)() @safe
if(id == TokenID.DocumentStart || id == TokenID.DocumentEnd)
{
//Set indentation to -1 .
unwindIndent(-1);
//Reset simple keys. Note that there can't be a block collection after '---'.
removePossibleSimpleKey();
allowSimpleKey_ = false;
Mark startMark = reader_.mark;
reader_.forward(3);
tokens_.push(simpleToken!id(startMark, reader_.mark));
}
///Aliases to add DOCUMENT-START or DOCUMENT-END token.
alias fetchDocumentIndicator!(TokenID.DocumentStart) fetchDocumentStart;
alias fetchDocumentIndicator!(TokenID.DocumentEnd) fetchDocumentEnd;
///Add FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
void fetchFlowCollectionStart(TokenID id)() @trusted
{
//'[' and '{' may start a simple key.
savePossibleSimpleKey();
//Simple keys are allowed after '[' and '{'.
allowSimpleKey_ = true;
++flowLevel_;
Mark startMark = reader_.mark;
reader_.forward();
tokens_.push(simpleToken!id(startMark, reader_.mark));
}
///Aliases to add FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
alias fetchFlowCollectionStart!(TokenID.FlowSequenceStart) fetchFlowSequenceStart;
alias fetchFlowCollectionStart!(TokenID.FlowMappingStart) fetchFlowMappingStart;
///Add FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
void fetchFlowCollectionEnd(TokenID id)() @safe
{
//Reset possible simple key on the current level.
removePossibleSimpleKey();
//No simple keys after ']' and '}'.
allowSimpleKey_ = false;
--flowLevel_;
Mark startMark = reader_.mark;
reader_.forward();
tokens_.push(simpleToken!id(startMark, reader_.mark));
}
///Aliases to add FLOW-SEQUENCE-START or FLOW-MAPPING-START token/
alias fetchFlowCollectionEnd!(TokenID.FlowSequenceEnd) fetchFlowSequenceEnd;
alias fetchFlowCollectionEnd!(TokenID.FlowMappingEnd) fetchFlowMappingEnd;
///Add FLOW-ENTRY token;
void fetchFlowEntry() @safe
{
//Reset possible simple key on the current level.
removePossibleSimpleKey();
//Simple keys are allowed after ','.
allowSimpleKey_ = true;
Mark startMark = reader_.mark;
reader_.forward();
tokens_.push(flowEntryToken(startMark, reader_.mark));
}
/**
* Additional checks used in block context in fetchBlockEntry and fetchKey.
*
* Params: type = String representing the token type we might need to add.
* id = Token type we might need to add.
*/
void blockChecks(string type, TokenID id)() @safe
{
//Are we allowed to start a key (not neccesarily a simple one)?
enforce(allowSimpleKey_, new Error(type ~ " keys are not allowed here",
reader_.mark));
if(addIndent(reader_.column))
{
tokens_.push(simpleToken!id(reader_.mark, reader_.mark));
}
}
///Add BLOCK-ENTRY token. Might add BLOCK-SEQUENCE-START in the process.
void fetchBlockEntry() @safe
{
if(flowLevel_ == 0){blockChecks!("Sequence", TokenID.BlockSequenceStart)();}
//It's an error for the block entry to occur in the flow context,
//but we let the parser detect this.
//Reset possible simple key on the current level.
removePossibleSimpleKey();
//Simple keys are allowed after '-'.
allowSimpleKey_ = true;
Mark startMark = reader_.mark;
reader_.forward();
tokens_.push(blockEntryToken(startMark, reader_.mark));
}
///Add KEY token. Might add BLOCK-MAPPING-START in the process.
void fetchKey() @safe
{
if(flowLevel_ == 0){blockChecks!("Mapping", TokenID.BlockMappingStart)();}
//Reset possible simple key on the current level.
removePossibleSimpleKey();
//Simple keys are allowed after '?' in the block context.
allowSimpleKey_ = (flowLevel_ == 0);
Mark startMark = reader_.mark;
reader_.forward();
tokens_.push(keyToken(startMark, reader_.mark));
}
///Add VALUE token. Might add KEY and/or BLOCK-MAPPING-START in the process.
void fetchValue() @safe
{
//Do we determine a simple key?
if(possibleSimpleKeys_.length > flowLevel_ &&
!possibleSimpleKeys_[flowLevel_].isNull)
{
const key = possibleSimpleKeys_[flowLevel_];
possibleSimpleKeys_[flowLevel_].isNull = true;
Mark keyMark = Mark(key.line, key.column);
const idx = key.tokenIndex - tokensTaken_;
assert(idx >= 0);
//Add KEY.
//Manually inserting since tokens are immutable (need linked list).
tokens_.insert(keyToken(keyMark, keyMark), idx);
//If this key starts a new block mapping, we need to add BLOCK-MAPPING-START.
if(flowLevel_ == 0 && addIndent(key.column))
{
tokens_.insert(blockMappingStartToken(keyMark, keyMark), idx);
}
//There cannot be two simple keys in a row.
allowSimpleKey_ = false;
}
//Part of a complex key
else
{
//We can start a complex value if and only if we can start a simple key.
enforce(flowLevel_ > 0 || allowSimpleKey_,
new Error("Mapping values are not allowed here", reader_.mark));
//If this value starts a new block mapping, we need to add
//BLOCK-MAPPING-START. It'll be detected as an error later by the parser.
if(flowLevel_ == 0 && addIndent(reader_.column))
{
tokens_.push(blockMappingStartToken(reader_.mark, reader_.mark));
}
//Reset possible simple key on the current level.
removePossibleSimpleKey();
//Simple keys are allowed after ':' in the block context.
allowSimpleKey_ = (flowLevel_ == 0);
}
//Add VALUE.
Mark startMark = reader_.mark;
reader_.forward();
tokens_.push(valueToken(startMark, reader_.mark));
}
///Add ALIAS or ANCHOR token.
void fetchAnchor_(TokenID id)() @trusted
if(id == TokenID.Alias || id == TokenID.Anchor)
{
//ALIAS/ANCHOR could be a simple key.
savePossibleSimpleKey();
//No simple keys after ALIAS/ANCHOR.
allowSimpleKey_ = false;
tokens_.push(scanAnchor(id));
}
///Aliases to add ALIAS or ANCHOR token.
alias fetchAnchor_!(TokenID.Alias) fetchAlias;
alias fetchAnchor_!(TokenID.Anchor) fetchAnchor;
///Add TAG token.
void fetchTag() @trusted
{
//TAG could start a simple key.
savePossibleSimpleKey();
//No simple keys after TAG.
allowSimpleKey_ = false;
tokens_.push(scanTag());
}
///Add block SCALAR token.
void fetchBlockScalar(ScalarStyle style)() @trusted
if(style == ScalarStyle.Literal || style == ScalarStyle.Folded)
{
//Reset possible simple key on the current level.
removePossibleSimpleKey();
//A simple key may follow a block scalar.
allowSimpleKey_ = true;
tokens_.push(scanBlockScalar(style));
}
///Aliases to add literal or folded block scalar.
alias fetchBlockScalar!(ScalarStyle.Literal) fetchLiteral;
alias fetchBlockScalar!(ScalarStyle.Folded) fetchFolded;
///Add quoted flow SCALAR token.
void fetchFlowScalar(ScalarStyle quotes)() @trusted
{
//A flow scalar could be a simple key.
savePossibleSimpleKey();
//No simple keys after flow scalars.
allowSimpleKey_ = false;
//Scan and add SCALAR.
tokens_.push(scanFlowScalar(quotes));
}
///Aliases to add single or double quoted block scalar.
alias fetchFlowScalar!(ScalarStyle.SingleQuoted) fetchSingle;
alias fetchFlowScalar!(ScalarStyle.DoubleQuoted) fetchDouble;
///Add plain SCALAR token.
void fetchPlain() @trusted
{
//A plain scalar could be a simple key
savePossibleSimpleKey();
//No simple keys after plain scalars. But note that scanPlain() will
//change this flag if the scan is finished at the beginning of the line.
allowSimpleKey_ = false;
//Scan and add SCALAR. May change allowSimpleKey_
tokens_.push(scanPlain());
}
///Check if the next token is DIRECTIVE: ^ '%' ...
bool checkDirective() @safe
{
return reader_.peek() == '%' && reader_.column == 0;
}
///Check if the next token is DOCUMENT-START: ^ '---' (' '|'\n')
bool checkDocumentStart() @safe
{
//Check one char first, then all 3, to prevent reading outside stream.
return reader_.column == 0 &&
reader_.peek() == '-' &&
reader_.prefix(3) == "---" &&
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(3));
}
///Check if the next token is DOCUMENT-END: ^ '...' (' '|'\n')
bool checkDocumentEnd() @safe
{
//Check one char first, then all 3, to prevent reading outside stream.
return reader_.column == 0 &&
reader_.peek() == '.' &&
reader_.prefix(3) == "..." &&
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(3));
}
///Check if the next token is BLOCK-ENTRY: '-' (' '|'\n')
bool checkBlockEntry() @safe
{
return reader_.peek() == '-' &&
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(1));
}
/**
* Check if the next token is KEY(flow context): '?'
*
* or KEY(block context): '?' (' '|'\n')
*/
bool checkKey() @safe
{
return reader_.peek() == '?' &&
(flowLevel_ > 0 ||
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(1)));
}
/**
* Check if the next token is VALUE(flow context): ':'
*
* or VALUE(block context): ':' (' '|'\n')
*/
bool checkValue() @safe
{
return reader_.peek() == ':' &&
(flowLevel_ > 0 ||
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(1)));
}
/**
* Check if the next token is a plain scalar.
*
* A plain scalar may start with any non-space character except:
* '-', '?', ':', ',', '[', ']', '{', '}',
* '#', '&', '*', '!', '|', '>', '\'', '\"',
* '%', '@', '`'.
*
* It may also start with
* '-', '?', ':'
* if it is followed by a non-space character.
*
* Note that we limit the last rule to the block context (except the
* '-' character) because we want the flow context to be space
* independent.
*/
bool checkPlain() @safe
{
const c = reader_.peek();
return !("-?:,[]{}#&*!|>\'\"%@` \t\0\n\r\u0085\u2028\u2029"d.canFind(c)) ||
(!" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(1)) &&
(c == '-' || (flowLevel_ == 0 && "?:"d.canFind(c))));
}
///Move to the next non-space character.
void findNextNonSpace() @safe
{
while(reader_.peek() == ' '){reader_.forward();}
}
///Scan a string of alphanumeric or "-_" characters.
dstring scanAlphaNumeric(string name)(const Mark startMark) @trusted
{
uint length = 0;
dchar c = reader_.peek();
while(isAlphaNum(c) || "-_"d.canFind(c))
{
++length;
c = reader_.peek(length);
}
enforce(length > 0,
new Error("While scanning " ~ name, startMark,
"expected alphanumeric, - or _, but found " ~ to!string(c),
reader_.mark));
return reader_.get(length);
}
///Scan all characters until nex line break.
dstring scanToNextBreak() @safe
{
uint length = 0;
while(!"\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(length))){++length;}
return reader_.get(length);
}
/**
* Move to next token in the file/stream.
*
* We ignore spaces, line breaks and comments.
* If we find a line break in the block context, we set
* allowSimpleKey` on.
*
* We do not yet support BOM inside the stream as the
* specification requires. Any such mark will be considered as a part
* of the document.
*/
void scanToNextToken() @safe
{
//TODO(PyYAML): We need to make tab handling rules more sane. A good rule is:
// Tabs cannot precede tokens
// BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END,
// KEY(block), VALUE(block), BLOCK-ENTRY
//So the checking code is
// if <TAB>:
// allowSimpleKey_ = false
//We also need to add the check for `allowSimpleKey_ == true` to
//`unwindIndent` before issuing BLOCK-END.
//Scanners for block, flow, and plain scalars need to be modified.
for(;;)
{
findNextNonSpace();
if(reader_.peek() == '#'){scanToNextBreak();}
if(scanLineBreak() != '\0')
{
if(flowLevel_ == 0){allowSimpleKey_ = true;}
}
else{break;}
}
}
///Scan directive token.
Token scanDirective() @trusted
{
Mark startMark = reader_.mark;
//Skip the '%'.
reader_.forward();
const name = scanDirectiveName(startMark);
const value = name == "YAML" ? scanYAMLDirectiveValue(startMark):
name == "TAG" ? scanTagDirectiveValue(startMark) : "";
Mark endMark = reader_.mark;
if(!["YAML"d, "TAG"d].canFind(name)){scanToNextBreak();}
scanDirectiveIgnoredLine(startMark);
//Storing directive name and value in a single string, separated by zero.
return directiveToken(startMark, endMark, to!string(name ~ '\0' ~ value));
}
///Scan name of a directive token.
dstring scanDirectiveName(const Mark startMark) @trusted
{
//Scan directive name.
const name = scanAlphaNumeric!"a directive"(startMark);
enforce(" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()),
new Error("While scanning a directive", startMark,
"expected alphanumeric, - or _, but found "
~ to!string(reader_.peek()), reader_.mark));
return name;
}
///Scan value of a YAML directive token. Returns major, minor version separated by '.'.
dstring scanYAMLDirectiveValue(const Mark startMark) @trusted
{
findNextNonSpace();
dstring result = scanYAMLDirectiveNumber(startMark);
enforce(reader_.peek() == '.',
new Error("While scanning a directive", startMark,
"expected a digit or '.', but found: "
~ to!string(reader_.peek()), reader_.mark));
//Skip the '.'.
reader_.forward();
result ~= '.' ~ scanYAMLDirectiveNumber(startMark);
enforce(" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()),
new Error("While scanning a directive", startMark,
"expected a digit or '.', but found: "
~ to!string(reader_.peek()), reader_.mark));
return result;
}
///Scan a number from a YAML directive.
dstring scanYAMLDirectiveNumber(const Mark startMark) @trusted
{
enforce(isDigit(reader_.peek()),
new Error("While scanning a directive", startMark,
"expected a digit, but found: " ~
to!string(reader_.peek()), reader_.mark));
//Already found the first digit in the enforce(), so set length to 1.
uint length = 1;
while(isDigit(reader_.peek(length))){++length;}
return reader_.get(length);
}
///Scan value of a tag directive.
dstring scanTagDirectiveValue(const Mark startMark) @safe
{
findNextNonSpace();
const handle = scanTagDirectiveHandle(startMark);
findNextNonSpace();
return handle ~ '\0' ~ scanTagDirectivePrefix(startMark);
}
///Scan handle of a tag directive.
dstring scanTagDirectiveHandle(const Mark startMark) @trusted
{
const value = scanTagHandle("directive", startMark);
enforce(reader_.peek() == ' ',
new Error("While scanning a directive handle", startMark,
"expected ' ', but found: " ~ to!string(reader_.peek()),
reader_.mark));
return value;
}
///Scan prefix of a tag directive.
dstring scanTagDirectivePrefix(const Mark startMark) @trusted
{
const value = scanTagURI("directive", startMark);
enforce(" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()),
new Error("While scanning a directive prefix", startMark,
"expected ' ', but found" ~ to!string(reader_.peek()),
reader_.mark));
return value;
}
///Scan (and ignore) ignored line after a directive.
void scanDirectiveIgnoredLine(const Mark startMark) @trusted
{
findNextNonSpace();
if(reader_.peek() == '#'){scanToNextBreak();}
enforce("\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()),
new Error("While scanning a directive", startMark,
"expected comment or a line break, but found"
~ to!string(reader_.peek()), reader_.mark));
scanLineBreak();
}
/**
* Scan an alias or an anchor.
*
* The specification does not restrict characters for anchors and
* aliases. This may lead to problems, for instance, the document:
* [ *alias, value ]
* can be interpteted in two ways, as
* [ "value" ]
* and
* [ *alias , "value" ]
* Therefore we restrict aliases to ASCII alphanumeric characters.
*/
Token scanAnchor(TokenID id) @trusted
{
const startMark = reader_.mark;
const dchar i = reader_.get();
dstring value = i == '*' ? scanAlphaNumeric!("an alias")(startMark)
: scanAlphaNumeric!("an anchor")(startMark);
enforce((" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()) ||
("?:,]}%@").canFind(reader_.peek())),
new Error("While scanning an " ~ (i == '*') ? "alias" : "anchor",
startMark, "expected alphanumeric, - or _, but found "~
to!string(reader_.peek()), reader_.mark));
if(id == TokenID.Alias)
{
return aliasToken(startMark, reader_.mark, to!string(value));
}
else if(id == TokenID.Anchor)
{
return anchorToken(startMark, reader_.mark, to!string(value));
}
assert(false, "This code should never be reached");
}
///Scan a tag token.
Token scanTag() @trusted
{
const startMark = reader_.mark;
dchar c = reader_.peek(1);
dstring handle = "";
dstring suffix;
if(c == '<')
{
reader_.forward(2);
suffix = scanTagURI("tag", startMark);
enforce(reader_.peek() == '>',
new Error("While scanning a tag", startMark,
"expected '>' but found" ~ to!string(reader_.peek()),
reader_.mark));
reader_.forward();
}
else if(" \t\0\n\r\u0085\u2028\u2029"d.canFind(c))
{
suffix = "!";
reader_.forward();
}
else
{
uint length = 1;
bool useHandle = false;
while(!" \0\n\r\u0085\u2028\u2029"d.canFind(c))
{
if(c == '!')
{
useHandle = true;
break;
}
++length;
c = reader_.peek(length);
}
if(useHandle){handle = scanTagHandle("tag", startMark);}
else
{
handle = "!";
reader_.forward();
}
suffix = scanTagURI("tag", startMark);
}
enforce(" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()),
new Error("While scanning a tag", startMark,
"expected ' ' but found" ~ to!string(reader_.peek()),
reader_.mark));
return tagToken(startMark, reader_.mark, to!string(handle ~ '\0' ~ suffix));
}
///Scan a block scalar token with specified style.
Token scanBlockScalar(const ScalarStyle style) @system
{
const startMark = reader_.mark;
//Scan the header.
reader_.forward();
const indicators = scanBlockScalarIndicators(startMark);
const chomping = indicators[0];
const increment = indicators[1];
scanBlockScalarIgnoredLine(startMark);
//Determine the indentation level and go to the first non-empty line.
Mark endMark;
dchar[] breaks;
uint indent = max(1, indent_ + 1);
if(increment == int.min)
{
auto indentation = scanBlockScalarIndentation();
breaks = indentation[0];
endMark = indentation[2];
indent = max(indent, indentation[1]);
}
else
{
indent += increment - 1;
auto scalarBreaks = scanBlockScalarBreaks(indent);
breaks = scalarBreaks[0];
endMark = scalarBreaks[1];
}
dstring lineBreak = "";
//Using appender_, so clear it when we're done.
scope(exit){appender_.clear();}
//Scan the inner part of the block scalar.
while(reader_.column == indent && reader_.peek() != '\0')
{
appender_.put(breaks);
const bool leadingNonSpace = !" \t"d.canFind(reader_.peek());
appender_.put(scanToNextBreak());
lineBreak = ""d ~ scanLineBreak();
auto scalarBreaks = scanBlockScalarBreaks(indent);
breaks = scalarBreaks[0];
endMark = scalarBreaks[1];
if(reader_.column == indent && reader_.peek() != '\0')
{
//Unfortunately, folding rules are ambiguous.
//This is the folding according to the specification:
if(style == ScalarStyle.Folded && lineBreak == "\n" &&
leadingNonSpace && !" \t"d.canFind(reader_.peek()))
{
if(breaks.length == 0){appender_.put(' ');}
}
else{appender_.put(lineBreak);}
////this is Clark Evans's interpretation (also in the spec
////examples):
//
//if(style == ScalarStyle.Folded && lineBreak == "\n"d)
//{
// if(breaks.length == 0)
// {
// if(!" \t"d.canFind(reader_.peek())){appender_.put(' ');}
// else{chunks ~= lineBreak;}
// }
//}
//else{appender_.put(lineBreak);}
}
else{break;}
}
if(chomping != Chomping.Strip){appender_.put(lineBreak);}
if(chomping == Chomping.Keep){appender_.put(breaks);}
return scalarToken(startMark, endMark, to!string(cast(dstring)appender_.data), style);
}
///Scan chomping and indentation indicators of a scalar token.
Tuple!(Chomping, int) scanBlockScalarIndicators(const Mark startMark) @trusted
{
auto chomping = Chomping.Clip;
int increment = int.min;
dchar c = reader_.peek();
///Get chomping indicator, if detected. Return false otherwise.
bool getChomping()
{
if(!"+-"d.canFind(c)){return false;}
chomping = c == '+' ? Chomping.Keep : Chomping.Strip;
reader_.forward();
c = reader_.peek();
return true;
}
///Get increment indicator, if detected. Return false otherwise.
bool getIncrement()
{
if(!isDigit(c)){return false;}
increment = to!int(""d ~ c);
enforce(increment != 0,
new Error("While scanning a block scalar", startMark,
"expected indentation indicator in range 1-9, but found 0",
reader_.mark));
reader_.forward();
c = reader_.peek();
return true;
}
///Indicators can be in any order.
if(getChomping()) {getIncrement();}
else if(getIncrement()){getChomping();}
enforce(" \0\n\r\u0085\u2028\u2029"d.canFind(c),
new Error("While scanning a block scalar", startMark,
"expected chomping or indentation indicator, but found "
~ to!string(c), reader_.mark));
return tuple(chomping, increment);
}
///Scan (and ignore) ignored line in a block scalar.
void scanBlockScalarIgnoredLine(const Mark startMark) @trusted
{
findNextNonSpace();
if(reader_.peek()== '#'){scanToNextBreak();}
enforce("\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()),
new Error("While scanning a block scalar", startMark,
"expected a comment or a line break, but found "
~ to!string(reader_.peek()), reader_.mark));
scanLineBreak();
}
///Scan indentation in a block scalar, returning line breaks, max indent and end mark.
Tuple!(dchar[], uint, Mark) scanBlockScalarIndentation() @safe
{
dchar[] chunks;
uint maxIndent;
Mark endMark = reader_.mark;
while(" \n\r\u0085\u2028\u2029"d.canFind(reader_.peek()))
{
if(reader_.peek() != ' ')
{
chunks ~= scanLineBreak();
endMark = reader_.mark;
continue;
}
reader_.forward();
maxIndent = max(reader_.column, maxIndent);
}
return tuple(chunks, maxIndent, endMark);
}
///Scan line breaks at lower or specified indentation in a block scalar.
Tuple!(dchar[], Mark) scanBlockScalarBreaks(const uint indent) @safe
{
dchar[] chunks;
Mark endMark = reader_.mark;
for(;;)
{
while(reader_.column < indent && reader_.peek() == ' '){reader_.forward();}
if(!"\n\r\u0085\u2028\u2029"d.canFind(reader_.peek())){break;}
chunks ~= scanLineBreak();
endMark = reader_.mark;
}
return tuple(chunks, endMark);
}
///Scan a qouted flow scalar token with specified quotes.
Token scanFlowScalar(const ScalarStyle quotes) @system
{
const startMark = reader_.mark;
const quote = reader_.get();
//Using appender_, so clear it when we're done.
scope(exit){appender_.clear();}
//Puts scanned data to appender_.
scanFlowScalarNonSpaces(quotes, startMark);
while(reader_.peek() != quote)
{
//Puts scanned data to appender_.
scanFlowScalarSpaces(startMark);
scanFlowScalarNonSpaces(quotes, startMark);
}
reader_.forward();
return scalarToken(startMark, reader_.mark, to!string(cast(dstring)appender_.data), quotes);
}
///Scan nonspace characters in a flow scalar.
void scanFlowScalarNonSpaces(const ScalarStyle quotes, const Mark startMark) @system
{
for(;;)
{
dchar c = reader_.peek();
uint length = 0;
mixin FastCharSearch!" \t\0\n\r\u0085\u2028\u2029\'\"\\"d search;
//This is an optimized way of writing:
//while(!search.canFind(reader_.peek(length))){++length;}
outer: for(;;)
{
const slice = reader_.slice(length, length + 32);
enforce(slice.length > 0,
new Error("While reading a flow scalar", startMark,
"reached end of file", reader_.mark));
foreach(ch; slice)
{
if(search.canFind(ch)){break outer;}
++length;
}
}
appender_.put(reader_.prefix(length));
reader_.forward(length);
c = reader_.peek();
if(quotes == ScalarStyle.SingleQuoted &&
c == '\'' && reader_.peek(1) == '\'')
{
appender_.put('\'');
reader_.forward(2);
}
else if((quotes == ScalarStyle.DoubleQuoted && c == '\'') ||
(quotes == ScalarStyle.SingleQuoted && "\"\\"d.canFind(c)))
{
appender_.put(c);
reader_.forward();
}
else if(quotes == ScalarStyle.DoubleQuoted && c == '\\')
{
reader_.forward();
c = reader_.peek();
if((c in dyaml.escapes.fromEscapes) !is null)
{
appender_.put(dyaml.escapes.fromEscapes[c]);
reader_.forward();
}
else if((c in dyaml.escapes.escapeHexCodes) !is null)
{
length = dyaml.escapes.escapeHexCodes[c];
reader_.forward();
foreach(i; 0 .. length)
{
enforce(isHexDigit(reader_.peek(i)),
new Error(
"While scanning a double qouted scalar", startMark,
"expected escape sequence of " ~ to!string(length) ~
" hexadecimal numbers, but found " ~
to!string(reader_.peek(i)), reader_.mark));
}
dstring hex = reader_.get(length);
appender_.put(cast(dchar)parse!int(hex, 16));
}
else if("\n\r\u0085\u2028\u2029"d.canFind(c))
{
scanLineBreak();
appender_.put(scanFlowScalarBreaks(startMark));
}
else
{
throw new Error("While scanning a double quoted scalar", startMark,
"found unknown escape character: " ~ to!string(c),
reader_.mark);
}
}
else
{
return;
}
}
}
///Scan space characters in a flow scalar.
void scanFlowScalarSpaces(const Mark startMark) @system
{
uint length = 0;
while(" \t"d.canFind(reader_.peek(length))){++length;}
const whitespaces = reader_.prefix(length + 1);
const c = whitespaces[$ - 1];
enforce(c != '\0', new Error("While scanning a quoted scalar", startMark,
"found unexpected end of stream", reader_.mark));
if("\n\r\u0085\u2028\u2029"d.canFind(c))
{
reader_.forward(length);
const lineBreak = scanLineBreak();
const breaks = scanFlowScalarBreaks(startMark);
if(lineBreak != '\n'){appender_.put(lineBreak);}
else if(breaks.length == 0){appender_.put(' ');}
appender_.put(breaks);
}
else
{
appender_.put(whitespaces[0 .. $ - 1]);
reader_.forward(length);
}
}
///Scan line breaks in a flow scalar.
dstring scanFlowScalarBreaks(const Mark startMark) @system
{
auto appender = appender!dstring();
for(;;)
{
//Instead of checking indentation, we check for document separators.
const prefix = reader_.prefix(3);
if((prefix == "---"d || prefix == "..."d) &&
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(3)))
{
throw new Error("While scanning a quoted scalar", startMark,
"found unexpected document separator", reader_.mark);
}
while(" \t"d.canFind(reader_.peek())){reader_.forward();}
if("\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()))
{
appender.put(scanLineBreak());
}
else{return appender.data;}
}
}
///Scan plain scalar token (no block, no quotes).
Token scanPlain() @system
{
//We keep track of the allowSimpleKey_ flag here.
//Indentation rules are loosed for the flow context
//Using appender_, so clear it when we're done.
scope(exit){appender_.clear();}
const startMark = reader_.mark;
Mark endMark = startMark;
const indent = indent_ + 1;
//We allow zero indentation for scalars, but then we need to check for
//document separators at the beginning of the line.
//if(indent == 0){indent = 1;}
dstring spaces;
mixin FastCharSearch!" \t\0\n\r\u0085\u2028\u2029"d search;
for(;;)
{
if(reader_.peek() == '#'){break;}
uint length = 0;
dchar c;
for(;;)
{
c = reader_.peek(length);
const bool done = search.canFind(c) || (flowLevel_ == 0 && c == ':' &&
search.canFind(reader_.peek(length + 1))) ||
(flowLevel_ > 0 && ",:?[]{}"d.canFind(c));
if(done){break;}
++length;
}
//It's not clear what we should do with ':' in the flow context.
if(flowLevel_ > 0 && c == ':' &&
!search.canFind(reader_.peek(length + 1)) &&
!",[]{}"d.canFind(reader_.peek(length + 1)))
{
reader_.forward(length);
throw new Error("While scanning a plain scalar", startMark,
"found unexpected ':' . Please check "
"http://pyyaml.org/wiki/YAMLColonInFlowContext "
"for details.", reader_.mark);
}
if(length == 0){break;}
allowSimpleKey_ = false;
appender_.put(spaces);
appender_.put(reader_.get(length));
endMark = reader_.mark;
spaces = scanPlainSpaces(startMark);
if(spaces.length == 0 || reader_.peek() == '#' ||
(flowLevel_ == 0 && reader_.column < indent))
{
break;
}
}
return scalarToken(startMark, endMark, to!string(cast(dstring)appender_.data), ScalarStyle.Plain);
}
///Scan spaces in a plain scalar.
dstring scanPlainSpaces(const Mark startMark) @system
{
///The specification is really confusing about tabs in plain scalars.
///We just forbid them completely. Do not use tabs in YAML!
auto appender = appender!dstring();
uint length = 0;
while(reader_.peek(length) == ' '){++length;}
dstring whitespaces = reader_.get(length);
dchar c = reader_.peek();
if("\n\r\u0085\u2028\u2029"d.canFind(c))
{
const lineBreak = scanLineBreak();
allowSimpleKey_ = true;
bool end()
{
return ["---"d, "..."d].canFind(reader_.prefix(3)) &&
" \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(3));
}
if(end()){return "";}
dstring breaks;
while(" \n\r\u0085\u2028\u2029"d.canFind(reader_.peek()))
{
if(reader_.peek() == ' '){reader_.forward();}
else
{
breaks ~= scanLineBreak();
if(end()){return "";}
}
}
if(lineBreak != '\n'){appender.put(lineBreak);}
else if(breaks.length == 0){appender.put(' ');}
appender.put(breaks);
}
else if(whitespaces.length > 0)
{
appender.put(whitespaces);
}
return appender.data;
}
///Scan handle of a tag token.
dstring scanTagHandle(const string name, const Mark startMark) @system
{
dchar c = reader_.peek();
enforce(c == '!',
new Error("While scanning a " ~ name, startMark,
"expected a '!', but found: " ~ to!string(c), reader_.mark));
uint length = 1;
c = reader_.peek(length);
if(c != ' ')
{
while(isAlphaNum(c) || "-_"d.canFind(c))
{
++length;
c = reader_.peek(length);
}
if(c != '!')
{
reader_.forward(length);
throw new Error("While scanning a " ~ name, startMark,
"expected a '!', but found: " ~ to!string(c),
reader_.mark);
}
++length;
}
return reader_.get(length);
}
///Scan URI in a tag token.
dstring scanTagURI(const string name, const Mark startMark) @system
{
//Note: we do not check if URI is well-formed.
//Using appender_, so clear it when we're done.
scope(exit){appender_.clear();}
uint length = 0;
dchar c = reader_.peek();
while(isAlphaNum(c) || "-;/?:@&=+$,_.!~*\'()[]%"d.canFind(c))
{
if(c == '%')
{
appender_.put(reader_.get(length));
length = 0;
appender_.put(scanURIEscapes(name, startMark));
}
else{++length;}
c = reader_.peek(length);
}
if(length > 0)
{
appender_.put(reader_.get(length));
length = 0;
}
enforce(appender_.data.length > 0,
new Error("While parsing a " ~ name, startMark,
"expected URI, but found: " ~ to!string(c), reader_.mark));
return cast(dstring)appender_.data;
}
///Scan URI escape sequences.
dstring scanURIEscapes(const string name, const Mark startMark) @system
{
ubyte[] bytes;
Mark mark = reader_.mark;
while(reader_.peek() == '%')
{
reader_.forward();
ubyte b = 0;
uint mult = 16;
//Converting 2 hexadecimal digits to a byte.
foreach(k; 0 .. 2)
{
const dchar c = reader_.peek(k);
enforce(isHexDigit(c),
new Error("While scanning a " ~ name, startMark,
"expected URI escape sequence of "
"2 hexadecimal numbers, but found: " ~
to!string(c), reader_.mark));
uint digit;
if(c - '0' < 10){digit = c - '0';}
else if(c - 'A' < 6){digit = c - 'A';}
else if(c - 'a' < 6){digit = c - 'a';}
else{assert(false);}
b += mult * digit;
mult /= 16;
}
bytes ~= b;
reader_.forward(2);
}
try{return to!dstring(cast(string)bytes);}
catch(ConvException e)
{
throw new Error("While scanning a " ~ name, startMark, e.msg, mark);
}
catch(UTFException e)
{
throw new Error("While scanning a " ~ name, startMark, e.msg, mark);
}
}
/**
* Scan a line break, if any.
*
* Transforms:
* '\r\n' : '\n'
* '\r' : '\n'
* '\n' : '\n'
* '\u0085' : '\n'
* '\u2028' : '\u2028'
* '\u2029 : '\u2029'
* no break : '\0'
*/
dchar scanLineBreak() @safe
{
const c = reader_.peek();
if(c == '\n' || c == '\r' || c == '\u0085')
{
if(reader_.prefix(2) == "\r\n"d){reader_.forward(2);}
else{reader_.forward();}
return '\n';
}
if(c == '\u2028' || c == '\u2029')
{
reader_.forward();
return c;
}
return '\0';
}
}