debian-packages/dyaml
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Scanner style.

Browse source
This commit is contained in:
Ferdinand Majerech 2014-07-26 23:29:55 +02:00
parent f76e4cfd02
commit d5663b1e57
1 changed files with 172 additions and 197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

167
source/dyaml/scanner.d
View file

@ -4,10 +4,8 @@
// (See accompanying file LICENSE_1_0.txt or copy at // (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt) // http://www.boost.org/LICENSE_1_0.txt)
/** /// YAML scanner.
* YAML scanner. /// Code based on PyYAML: http://www.pyyaml.org
* Code based on PyYAML: http://www.pyyaml.org
*/
module dyaml.scanner; module dyaml.scanner;
@ -34,36 +32,32 @@ import dyaml.style;
import dyaml.token; import dyaml.token;
package: package:
/** /// Scanner produces tokens of the following types:
* Scanner produces tokens of the following types: /// STREAM-START
* STREAM-START /// STREAM-END
* STREAM-END /// DIRECTIVE(name, value)
* DIRECTIVE(name, value) /// DOCUMENT-START
* DOCUMENT-START /// DOCUMENT-END
* DOCUMENT-END /// BLOCK-SEQUENCE-START
* BLOCK-SEQUENCE-START /// BLOCK-MAPPING-START
* BLOCK-MAPPING-START /// BLOCK-END
* BLOCK-END /// FLOW-SEQUENCE-START
* FLOW-SEQUENCE-START /// FLOW-MAPPING-START
* FLOW-MAPPING-START /// FLOW-SEQUENCE-END
* FLOW-SEQUENCE-END /// FLOW-MAPPING-END
* FLOW-MAPPING-END /// BLOCK-ENTRY
* BLOCK-ENTRY /// FLOW-ENTRY
* FLOW-ENTRY /// KEY
* KEY /// VALUE
* VALUE /// ALIAS(value)
* ALIAS(value) /// ANCHOR(value)
* ANCHOR(value) /// TAG(value)
* TAG(value) /// SCALAR(value, plain, style)
* SCALAR(value, plain, style)
*/
/** /// Marked exception thrown at scanner errors.
* Marked exception thrown at scanner errors. ///
* /// See_Also: MarkedYAMLException
* See_Also: MarkedYAMLException
*/
class ScannerException : MarkedYAMLException class ScannerException : MarkedYAMLException
{ {
mixin MarkedExceptionCtors; mixin MarkedExceptionCtors;
@ -182,17 +176,14 @@ final class Scanner
reader_ = null; reader_ = null;
} }
/** /// Check if the next token is one of specified types.
* Check if the next token is one of specified types. ///
* /// If no types are specified, checks if any tokens are left.
* If no types are specified, checks if any tokens are left. ///
* /// Params: ids = Token IDs to check for.
* Params: ids = Token IDs to check for. ///
* /// Returns: true if the next token is one of specified types, or if there are
* Returns: true if the next token is one of specified types, /// any tokens left if no types specified, false otherwise.
* or if there are any tokens left if no types specified.
* false otherwise.
*/
bool checkToken(const TokenID[] ids ...) @safe bool checkToken(const TokenID[] ids ...) @safe
{ {
//Check if the next token is one of specified types. //Check if the next token is one of specified types.
@ -212,11 +203,9 @@ final class Scanner
return false; return false;
} }
/** /// Return the next token, but keep it in the queue.
* Return the next token, but keep it in the queue. ///
* /// Must not be called if there are no tokens left.
* Must not be called if there are no tokens left.
*/
ref const(Token) peekToken() @safe ref const(Token) peekToken() @safe
{ {
while(needMoreTokens) { fetchToken(); } while(needMoreTokens) { fetchToken(); }
@ -224,11 +213,9 @@ final class Scanner
assert(false, "No token left to peek"); assert(false, "No token left to peek");
} }
/** /// Return the next token, removing it from the queue.
* Return the next token, removing it from the queue. ///
* /// Must not be called if there are no tokens left.
* Must not be called if there are no tokens left.
*/
Token getToken() @safe Token getToken() @safe
{ {
while(needMoreTokens){fetchToken();} while(needMoreTokens){fetchToken();}
@ -290,7 +277,7 @@ final class Scanner
/// Fetch at token, adding it to tokens_. /// Fetch at token, adding it to tokens_.
void fetchToken() @safe void fetchToken() @safe
{ {
///Eat whitespaces and comments until we reach the next token. // Eat whitespaces and comments until we reach the next token.
scanToNextToken(); scanToNextToken();
// Remove obsolete possible simple keys. // Remove obsolete possible simple keys.
@ -326,9 +313,9 @@ final class Scanner
if(c == '\"') { return fetchDouble(); } if(c == '\"') { return fetchDouble(); }
if(checkPlain()) { return fetchPlain(); } if(checkPlain()) { return fetchPlain(); }
throw new Error(format("While scanning for the next token, found " throw new ScannerException("While scanning for the next token, found character "
"character \'%s\', index %s that cannot start any token" "\'%s\', index %s that cannot start any token"
, c, to!int(c)), reader_.mark); .format(c, to!int(c)), reader_.mark);
} }
@ -344,15 +331,13 @@ final class Scanner
return minTokenNumber; return minTokenNumber;
} }
/** /// Remove entries that are no longer possible simple keys.
* Remove entries that are no longer possible simple keys. ///
* /// According to the YAML specification, simple keys
* According to the YAML specification, simple keys /// - should be limited to a single line,
* - should be limited to a single line, /// - should be no longer than 1024 characters.
* - should be no longer than 1024 characters. /// Disabling this will allow simple keys of any length and
* Disabling this will allow simple keys of any length and /// height (may cause problems if indentation is broken though).
* height (may cause problems if indentation is broken though).
*/
void stalePossibleSimpleKeys() @safe pure void stalePossibleSimpleKeys() @safe pure
{ {
foreach(level, ref key; possibleSimpleKeys_) foreach(level, ref key; possibleSimpleKeys_)
@ -361,7 +346,7 @@ final class Scanner
if(key.line != reader_.line || reader_.charIndex - key.charIndex > 1024) if(key.line != reader_.line || reader_.charIndex - key.charIndex > 1024)
{ {
enforce(!key.required, enforce(!key.required,
new Error("While scanning a simple key", new ScannerException("While scanning a simple key",
Mark(key.line, key.column), Mark(key.line, key.column),
"could not find expected ':'", reader_.mark)); "could not find expected ':'", reader_.mark));
key.isNull = true; key.isNull = true;
@ -369,11 +354,9 @@ final class Scanner
} }
} }
/** /// Check if the next token starts a possible simple key and if so, save its position.
* Check if the next token starts a possible simple key and if so, save its position. ///
* /// This function is called for ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
* This function is called for ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'.
*/
void savePossibleSimpleKey() @safe pure void savePossibleSimpleKey() @safe pure
{ {
// Check if a simple key is required at the current position. // Check if a simple key is required at the current position.
@ -389,11 +372,8 @@ final class Scanner
const line = reader_.line; const line = reader_.line;
const column = reader_.column; const column = reader_.column;
const key = SimpleKey(cast(uint)reader_.charIndex, const key = SimpleKey(cast(uint)reader_.charIndex, tokenCount, line,
tokenCount, cast(ushort)min(column, ushort.max), required);
line,
column < ushort.max ? cast(ushort)column : ushort.max,
required);
if(possibleSimpleKeys_.length <= flowLevel_) if(possibleSimpleKeys_.length <= flowLevel_)
{ {
@ -420,11 +400,9 @@ final class Scanner
} }
} }
/** /// Decrease indentation, removing entries in indents_.
* Decrease indentation, removing entries in indents_. ///
* /// Params: column = Current column in the file/stream.
* Params: column = Current column in the file/stream.
*/
void unwindIndent(const int column) @trusted void unwindIndent(const int column) @trusted
{ {
if(flowLevel_ > 0) if(flowLevel_ > 0)
@ -455,13 +433,11 @@ final class Scanner
} }
} }
/** /// Increase indentation if needed.
* Increase indentation if needed. ///
* /// Params: column = Current column in the file/stream.
* Params: column = Current column in the file/stream. ///
* /// Returns: true if the indentation was increased, false otherwise.
* Returns: true if the indentation was increased, false otherwise.
*/
bool addIndent(int column) @trusted bool addIndent(int column) @trusted
{ {
if(indent_ >= column){return false;} if(indent_ >= column){return false;}
@ -572,12 +548,10 @@ final class Scanner
tokens_.push(flowEntryToken(startMark, reader_.mark)); tokens_.push(flowEntryToken(startMark, reader_.mark));
} }
/** /// Additional checks used in block context in fetchBlockEntry and fetchKey.
* Additional checks used in block context in fetchBlockEntry and fetchKey. ///
* /// Params: type = String representing the token type we might need to add.
* Params: type = String representing the token type we might need to add. /// id = Token type we might need to add.
* id = Token type we might need to add.
*/
void blockChecks(string type, TokenID id)() @safe void blockChecks(string type, TokenID id)() @safe
{ {
//Are we allowed to start a key (not neccesarily a simple one)? //Are we allowed to start a key (not neccesarily a simple one)?
@ -1912,7 +1886,8 @@ final class Scanner
/// characters into that slice. /// characters into that slice.
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
void scanTagURIToSlice(string name)(const Mark startMark) @trusted pure nothrow void scanTagURIToSlice(string name)(const Mark startMark)
@trusted pure nothrow // @nogc
{ {
// Note: we do not check if URI is well-formed. // Note: we do not check if URI is well-formed.
dchar c = reader_.peek(); dchar c = reader_.peek();