debian-packages/dyaml
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Directive scanning is now fully UTF-8.

Browse source
This commit is contained in:
Ferdinand Majerech 2014-07-29 04:19:44 +02:00
parent 38143a2c64
commit 4a09338a7a
1 changed files with 22 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
source/dyaml/scanner.d
View file

@ -915,14 +915,14 @@ final class Scanner
if(error_) { return Token.init; } if(error_) { return Token.init; }
const name = reader_.sliceBuilder8.finish(); const name = reader_.sliceBuilder8.finish();
reader_.sliceBuilder.begin(); reader_.sliceBuilder8.begin();
// Index where tag handle ends and suffix starts in a tag directive value. // Index where tag handle ends and suffix starts in a tag directive value.
uint tagHandleEnd = uint.max; uint tagHandleEnd = uint.max;
if(name == "YAML") { scanYAMLDirectiveValueToSlice(startMark); } if(name == "YAML") { scanYAMLDirectiveValueToSlice8(startMark); }
else if(name == "TAG") { tagHandleEnd = scanTagDirectiveValueToSlice(startMark); } else if(name == "TAG") { tagHandleEnd = scanTagDirectiveValueToSlice8(startMark); }
if(error_) { return Token.init; } if(error_) { return Token.init; }
const value = reader_.sliceBuilder.finish(); const value = reader_.sliceBuilder8.finish();
Mark endMark = reader_.mark; Mark endMark = reader_.mark;
@ -935,11 +935,10 @@ final class Scanner
scanToNextBreak(); scanToNextBreak();
} }
scanDirectiveIgnoredLine(startMark); scanDirectiveIgnoredLine8(startMark);
if(error_) { return Token.init; } if(error_) { return Token.init; }
return directiveToken(startMark, endMark, utf32To8(value), return directiveToken(startMark, endMark, value, directive, tagHandleEnd);
directive, tagHandleEnd);
} }
/// Scan name of a directive token. /// Scan name of a directive token.
@ -965,12 +964,12 @@ final class Scanner
/// characters into that slice. /// characters into that slice.
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
void scanYAMLDirectiveValueToSlice(const Mark startMark) void scanYAMLDirectiveValueToSlice8(const Mark startMark)
@system pure nothrow @nogc @system pure nothrow @nogc
{ {
findNextNonSpace(); findNextNonSpace();
scanYAMLDirectiveNumberToSlice(startMark); scanYAMLDirectiveNumberToSlice8(startMark);
if(error_) { return; } if(error_) { return; }
if(reader_.peek() != '.') if(reader_.peek() != '.')
@ -982,8 +981,8 @@ final class Scanner
// Skip the '.'. // Skip the '.'.
reader_.forward(); reader_.forward();
reader_.sliceBuilder.write('.'); reader_.sliceBuilder8.write('.');
scanYAMLDirectiveNumberToSlice(startMark); scanYAMLDirectiveNumberToSlice8(startMark);
if(error_) { return; } if(error_) { return; }
if(!" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek())) if(!" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek()))
@ -999,7 +998,7 @@ final class Scanner
/// characters into that slice. /// characters into that slice.
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
void scanYAMLDirectiveNumberToSlice(const Mark startMark) void scanYAMLDirectiveNumberToSlice8(const Mark startMark)
@system pure nothrow @nogc @system pure nothrow @nogc
{ {
if(!isDigit(reader_.peek())) if(!isDigit(reader_.peek()))
@ -1013,7 +1012,7 @@ final class Scanner
uint length = 1; uint length = 1;
while(isDigit(reader_.peek(length))) { ++length; } while(isDigit(reader_.peek(length))) { ++length; }
reader_.sliceBuilder.write(reader_.get(length)); reader_.sliceBuilder8.write(reader_.get8(length));
} }
/// Scan value of a tag directive. /// Scan value of a tag directive.
@ -1024,16 +1023,16 @@ final class Scanner
/// Returns: Length of tag handle (which is before tag prefix) in scanned data /// Returns: Length of tag handle (which is before tag prefix) in scanned data
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
uint scanTagDirectiveValueToSlice(const Mark startMark) uint scanTagDirectiveValueToSlice8(const Mark startMark)
@system pure nothrow @system pure nothrow
{ {
findNextNonSpace(); findNextNonSpace();
const startLength = reader_.sliceBuilder.length; const startLength = reader_.sliceBuilder8.length;
scanTagDirectiveHandleToSlice(startMark); scanTagDirectiveHandleToSlice8(startMark);
if(error_) { return uint.max; } if(error_) { return uint.max; }
const handleLength = cast(uint)(reader_.sliceBuilder.length - startLength); const handleLength = cast(uint)(reader_.sliceBuilder8.length - startLength);
findNextNonSpace(); findNextNonSpace();
scanTagDirectivePrefixToSlice(startMark); scanTagDirectivePrefixToSlice8(startMark);
return handleLength; return handleLength;
} }
@ -1044,10 +1043,10 @@ final class Scanner
/// characters into that slice. /// characters into that slice.
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
void scanTagDirectiveHandleToSlice(const Mark startMark) void scanTagDirectiveHandleToSlice8(const Mark startMark)
@system pure nothrow @nogc @system pure nothrow @nogc
{ {
scanTagHandleToSlice!"directive"(startMark); scanTagHandleToSlice8!"directive"(startMark);
if(error_) { return; } if(error_) { return; }
if(reader_.peek() == ' ') { return; } if(reader_.peek() == ' ') { return; }
error("While scanning a directive handle", startMark, error("While scanning a directive handle", startMark,
@ -1060,9 +1059,9 @@ final class Scanner
/// characters into that slice. /// characters into that slice.
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
void scanTagDirectivePrefixToSlice(const Mark startMark) @system pure nothrow void scanTagDirectivePrefixToSlice8(const Mark startMark) @system pure nothrow
{ {
scanTagURIToSlice!"directive"(startMark); scanTagURIToSlice8!"directive"(startMark);
if(" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek())) { return; } if(" \0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek())) { return; }
error("While scanning a directive prefix", startMark, error("While scanning a directive prefix", startMark,
expected("' '", reader_.peek()), reader_.mark); expected("' '", reader_.peek()), reader_.mark);
@ -1071,7 +1070,7 @@ final class Scanner
/// Scan (and ignore) ignored line after a directive. /// Scan (and ignore) ignored line after a directive.
/// ///
/// In case of an error, error_ is set. Use throwIfError() to handle this. /// In case of an error, error_ is set. Use throwIfError() to handle this.
void scanDirectiveIgnoredLine(const Mark startMark) @safe pure nothrow @nogc void scanDirectiveIgnoredLine8(const Mark startMark) @safe pure nothrow @nogc
{ {
findNextNonSpace(); findNextNonSpace();
if(reader_.peek() == '#') { scanToNextBreak(); } if(reader_.peek() == '#') { scanToNextBreak(); }