debian-packages/dyaml
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

UTF-8 scanPlain and callees.

Browse source
This commit is contained in:
Ferdinand Majerech 2014-07-29 04:28:07 +02:00
parent d80917419f
commit 239152f793
1 changed files with 19 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
source/dyaml/scanner.d
View file

@ -1730,9 +1730,9 @@ final class Scanner
mixin FastCharSearch!" \t\0\n\r\u0085\u2028\u2029"d search; mixin FastCharSearch!" \t\0\n\r\u0085\u2028\u2029"d search;
reader_.sliceBuilder.begin(); reader_.sliceBuilder8.begin();
alias Transaction = SliceBuilder.Transaction; alias Transaction = SliceBuilder8.Transaction;
Transaction spacesTransaction; Transaction spacesTransaction;
// Stop at a comment. // Stop at a comment.
while(reader_.peek() != '#') while(reader_.peek() != '#')
@ -1757,7 +1757,7 @@ final class Scanner
{ {
// This is an error; throw the slice away. // This is an error; throw the slice away.
spacesTransaction.commit(); spacesTransaction.commit();
reader_.sliceBuilder.finish(); reader_.sliceBuilder8.finish();
reader_.forward(length); reader_.forward(length);
error("While scanning a plain scalar", startMark, error("While scanning a plain scalar", startMark,
"found unexpected ':' . Please check " "found unexpected ':' . Please check "
@ -1770,16 +1770,16 @@ final class Scanner
allowSimpleKey_ = false; allowSimpleKey_ = false;
reader_.sliceBuilder.write(reader_.get(length)); reader_.sliceBuilder8.write(reader_.get8(length));
endMark = reader_.mark; endMark = reader_.mark;
spacesTransaction.commit(); spacesTransaction.commit();
spacesTransaction = Transaction(reader_.sliceBuilder); spacesTransaction = Transaction(reader_.sliceBuilder8);
const startLength = reader_.sliceBuilder.length; const startLength = reader_.sliceBuilder8.length;
scanPlainSpacesToSlice(startMark); scanPlainSpacesToSlice8(startMark);
if(startLength == reader_.sliceBuilder.length || if(startLength == reader_.sliceBuilder8.length ||
(flowLevel_ == 0 && reader_.column < indent)) (flowLevel_ == 0 && reader_.column < indent))
{ {
break; break;
@ -1787,16 +1787,16 @@ final class Scanner
} }
spacesTransaction.__dtor(); spacesTransaction.__dtor();
const slice = reader_.sliceBuilder.finish(); const slice = reader_.sliceBuilder8.finish();
return scalarToken(startMark, endMark, slice.utf32To8, ScalarStyle.Plain); return scalarToken(startMark, endMark, slice, ScalarStyle.Plain);
} }
/// Scan spaces in a plain scalar. /// Scan spaces in a plain scalar.
/// ///
/// Assumes that the caller is building a slice in Reader, and puts the spaces /// Assumes that the caller is building a slice in Reader, and puts the spaces
/// into that slice. /// into that slice.
void scanPlainSpacesToSlice(const Mark startMark) @system pure nothrow @nogc void scanPlainSpacesToSlice8(const Mark startMark) @system pure nothrow @nogc
{ {
// The specification is really confusing about tabs in plain scalars. // The specification is really confusing about tabs in plain scalars.
// We just forbid them completely. Do not use tabs in YAML! // We just forbid them completely. Do not use tabs in YAML!
@ -1804,14 +1804,14 @@ final class Scanner
// Get as many plain spaces as there are. // Get as many plain spaces as there are.
size_t length = 0; size_t length = 0;
while(reader_.peek(length) == ' ') { ++length; } while(reader_.peek(length) == ' ') { ++length; }
dchar[] whitespaces = reader_.get(length); char[] whitespaces = reader_.get8(length);
dchar c = reader_.peek(); dchar c = reader_.peek();
// No newline after the spaces (if any) // No newline after the spaces (if any)
if(!"\n\r\u0085\u2028\u2029"d.canFind(c)) if(!"\n\r\u0085\u2028\u2029"d.canFind(c))
{ {
// We have spaces, but no newline. // We have spaces, but no newline.
if(whitespaces.length > 0) { reader_.sliceBuilder.write(whitespaces); } if(whitespaces.length > 0) { reader_.sliceBuilder8.write(whitespaces); }
return; return;
} }
@ -1821,7 +1821,7 @@ final class Scanner
static bool end(Reader reader_) @safe pure nothrow @nogc static bool end(Reader reader_) @safe pure nothrow @nogc
{ {
return ("---"d == reader_.prefix(3) || "..."d == reader_.prefix(3)) return ("---" == reader_.prefix8(3) || "..." == reader_.prefix8(3))
&& " \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(3)); && " \t\0\n\r\u0085\u2028\u2029"d.canFind(reader_.peek(3));
} }
@ -1829,9 +1829,9 @@ final class Scanner
bool extraBreaks = false; bool extraBreaks = false;
alias Transaction = SliceBuilder.Transaction; alias Transaction = SliceBuilder8.Transaction;
auto transaction = Transaction(reader_.sliceBuilder); auto transaction = Transaction(reader_.sliceBuilder8);
if(lineBreak != '\n') { reader_.sliceBuilder.write(lineBreak); } if(lineBreak != '\n') { reader_.sliceBuilder8.write(lineBreak); }
while(" \n\r\u0085\u2028\u2029"d.canFind(reader_.peek())) while(" \n\r\u0085\u2028\u2029"d.canFind(reader_.peek()))
{ {
if(reader_.peek() == ' ') { reader_.forward(); } if(reader_.peek() == ' ') { reader_.forward(); }
@ -1839,7 +1839,7 @@ final class Scanner
{ {
const lBreak = scanLineBreak(); const lBreak = scanLineBreak();
extraBreaks = true; extraBreaks = true;
reader_.sliceBuilder.write(lBreak); reader_.sliceBuilder8.write(lBreak);
if(end(reader_)) { return; } if(end(reader_)) { return; }
} }
@ -1847,7 +1847,7 @@ final class Scanner
transaction.commit(); transaction.commit();
// No line breaks, only a space. // No line breaks, only a space.
if(lineBreak == '\n' && !extraBreaks) { reader_.sliceBuilder.write(' '); } if(lineBreak == '\n' && !extraBreaks) { reader_.sliceBuilder8.write(' '); }
} }
/// Scan handle of a tag token. /// Scan handle of a tag token.