debian-packages/dyaml
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Implemented a simple Queue data structure, until Phobos has a

Browse source 
replacement. Tokens are not immutable anymore.
This commit is contained in:
Ferdinand Majerech 2011-10-20 10:34:34 +02:00
parent 595302fbff
commit fb814c66c2
5 changed files with 309 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
dyaml/emitter.d
View file

@ -30,6 +30,7 @@ import dyaml.event;
import dyaml.exception;
import dyaml.flags;
import dyaml.linebreak;
import dyaml.queue;
import dyaml.tag;
import dyaml.token;
@ -83,7 +84,7 @@ struct Emitter
//TODO Should be replaced by a queue or linked list once Phobos has anything usable.
///Event queue.
Event[] events_;
Queue!Event events_;
///Event we're currently emitting.
Event event_;
@ -171,21 +172,25 @@ struct Emitter
{
stream_ = null;
clear(states_);
states_ = null;
clear(events_);
clear(indents_);
indents_ = null;
clear(tagDirectives_);
tagDirectives_ = null;
clear(preparedAnchor_);
preparedAnchor_ = null;
clear(preparedTag_);
preparedTag_ = null;
}
///Emit an event. Throws EmitterException on error.
void emit(immutable Event event)
{
events_ ~= event;
events_.push(event);
while(!needMoreEvents())
{
event_ = events_.front;
events_.popFront();
event_ = events_.pop();
state_();
clear(event_);
}
@ -237,23 +242,31 @@ struct Emitter
}
///In some cases, we wait for a few next events before emitting.
bool needMoreEvents() const
bool needMoreEvents()
{
if(events_.length == 0){return true;}
if(events_[0].id == EventID.DocumentStart){return needEvents(1);}
if(events_[0].id == EventID.SequenceStart){return needEvents(2);}
if(events_[0].id == EventID.MappingStart) {return needEvents(3);}
immutable event = events_.peek();
if(event.id == EventID.DocumentStart){return needEvents(1);}
if(event.id == EventID.SequenceStart){return needEvents(2);}
if(event.id == EventID.MappingStart) {return needEvents(3);}
return false;
}
///Determines if we need specified number of more events.
bool needEvents(in uint count) const
bool needEvents(in uint count)
{
int level = 0;
foreach(ref event; events_[1 .. $])
//Rather ugly, but good enough for now.
//Couldn't be bothered writing a range as events_ should eventually
//become a Phobos queue/linked list.
events_.startIteration();
events_.next();
while(!events_.iterationOver())
{
immutable event = events_.next();
if([EventID.DocumentStart, EventID.SequenceStart,
EventID.MappingStart].canFind(event.id))
{
@ -645,14 +658,14 @@ struct Emitter
bool checkEmptySequence() const
{
return event_.id == EventID.SequenceStart && events_.length > 0
&& events_[0].id == EventID.SequenceEnd;
&& events_.peek().id == EventID.SequenceEnd;
}
///Check if an empty mapping is next.
bool checkEmptyMapping() const
{
return event_.id == EventID.MappingStart && events_.length > 0
&& events_[0].id == EventID.MappingEnd;
&& events_.peek().id == EventID.MappingEnd;
}
///Check if an empty document is next.
@ -663,7 +676,7 @@ struct Emitter
return false;
}
immutable event = events_[0];
immutable event = events_.peek();
bool emptyScalar = event.id == EventID.Scalar && event.anchor.isNull() &&
event.tag.isNull() && event.implicit && event.value == "";
return emptyScalar;

50
dyaml/parser.d
View file

@ -252,7 +252,7 @@ final class Parser
///Parse stream start.
Event parseStreamStart()
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
state_ = &parseImplicitDocumentStart;
return streamStartEvent(token.startMark, token.endMark, token.encoding);
}
@ -265,7 +265,7 @@ final class Parser
TokenID.StreamEnd))
{
tagHandles_ = defaultTags_;
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
states_ ~= &parseDocumentEnd;
state_ = &parseBlockNode;
@ -300,7 +300,7 @@ final class Parser
else
{
//Parse the end of the stream.
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
assert(states_.length == 0);
assert(marks_.length == 0);
state_ = null;
@ -342,7 +342,7 @@ final class Parser
//Process directives.
while(scanner_.checkToken(TokenID.Directive))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
//Name and value are separated by '\0'.
auto parts = token.value.split("\0");
const name = parts[0];
@ -416,7 +416,7 @@ final class Parser
{
if(scanner_.checkToken(TokenID.Alias))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
state_ = popState();
return aliasEvent(token.startMark, token.endMark, Anchor(token.value));
}
@ -431,7 +431,7 @@ final class Parser
{
if(!scanner_.checkToken(id)){return false;}
invalidMarks = false;
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(start){startMark = token.startMark;}
if(id == TokenID.Tag){tagMark = token.startMark;}
endMark = token.endMark;
@ -462,7 +462,7 @@ final class Parser
if(scanner_.checkToken(TokenID.Scalar))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
implicit = (token.style == ScalarStyle.Plain && tag is null) || tag == "!";
bool implicit_2 = (!implicit) && tag is null;
@ -515,7 +515,7 @@ final class Parser
[implicit, false] , "");
}
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
throw new Error("While parsing a " ~ (block ? "block" : "flow") ~ " node",
startMark, "expected the node content, but found: "
~ token.idString, token.startMark);
@ -571,7 +571,7 @@ final class Parser
if(scanner_.checkToken(TokenID.BlockEntry))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(!scanner_.checkToken(TokenID.BlockEntry, TokenID.BlockEnd))
{
states_~= &parseBlockSequenceEntry!false;
@ -584,7 +584,7 @@ final class Parser
if(!scanner_.checkToken(TokenID.BlockEnd))
{
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
throw new Error("While parsing a block collection", marks_[$ - 1],
"expected block end, but found " ~ token.idString,
token.startMark);
@ -592,7 +592,7 @@ final class Parser
state_ = popState();
popMark();
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
return sequenceEndEvent(token.startMark, token.endMark);
}
@ -603,7 +603,7 @@ final class Parser
{
if(scanner_.checkToken(TokenID.BlockEntry))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(!scanner_.checkToken(TokenID.BlockEntry, TokenID.Key,
TokenID.Value, TokenID.BlockEnd))
@ -617,7 +617,7 @@ final class Parser
}
state_ = popState();
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
return sequenceEndEvent(token.startMark, token.endMark);
}
@ -635,7 +635,7 @@ final class Parser
if(scanner_.checkToken(TokenID.Key))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(!scanner_.checkToken(TokenID.Key, TokenID.Value, TokenID.BlockEnd))
{
@ -649,7 +649,7 @@ final class Parser
if(!scanner_.checkToken(TokenID.BlockEnd))
{
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
throw new Error("While parsing a block mapping", marks_[$ - 1],
"expected block end, but found: " ~ token.idString,
token.startMark);
@ -657,7 +657,7 @@ final class Parser
state_ = popState();
popMark();
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
return mappingEndEvent(token.startMark, token.endMark);
}
@ -666,7 +666,7 @@ final class Parser
{
if(scanner_.checkToken(TokenID.Value))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(!scanner_.checkToken(TokenID.Key, TokenID.Value, TokenID.BlockEnd))
{
@ -710,7 +710,7 @@ final class Parser
}
else
{
Token token = scanner_.peekToken;
immutable token = scanner_.peekToken;
throw new Error("While parsing a flow sequence", marks_[$ - 1],
"expected ',' or ']', but got: " ~
token.idString, token.startMark);
@ -719,7 +719,7 @@ final class Parser
if(scanner_.checkToken(TokenID.Key))
{
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
state_ = &parseFlowSequenceEntryMappingKey;
return mappingStartEvent(token.startMark, token.endMark,
Anchor(), Tag(), true, CollectionStyle.Flow);
@ -731,7 +731,7 @@ final class Parser
}
}
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
state_ = popState();
popMark();
return sequenceEndEvent(token.startMark, token.endMark);
@ -740,7 +740,7 @@ final class Parser
///Parse a key in flow context.
Event parseFlowKey(in Event delegate() nextState)
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(!scanner_.checkToken(TokenID.Value, TokenID.FlowEntry,
TokenID.FlowSequenceEnd))
@ -764,7 +764,7 @@ final class Parser
{
if(scanner_.checkToken(TokenID.Value))
{
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
if(!scanner_.checkToken(TokenID.FlowEntry, checkId))
{
states_ ~= nextState;
@ -790,7 +790,7 @@ final class Parser
Event parseFlowSequenceEntryMappingEnd()
{
state_ = &parseFlowSequenceEntry!false;
Token token = scanner_.peekToken();
immutable token = scanner_.peekToken();
return mappingEndEvent(token.startMark, token.startMark);
}
@ -817,7 +817,7 @@ final class Parser
}
else
{
Token token = scanner_.peekToken;
immutable token = scanner_.peekToken;
throw new Error("While parsing a flow mapping", marks_[$ - 1],
"expected ',' or '}', but got: " ~
token.idString, token.startMark);
@ -836,7 +836,7 @@ final class Parser
}
}
Token token = scanner_.getToken();
immutable token = scanner_.getToken();
state_ = popState();
popMark();
return mappingEndEvent(token.startMark, token.endMark);

230
dyaml/queue.d Normal file
View file

@ -0,0 +1,230 @@
// Copyright Ferdinand Majerech 2011.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
module dyaml.queue;
///Queue collection.
import core.stdc.stdlib;
import core.memory;
import std.container;
package:
/**
* Simple queue implemented as a singly linked list with a tail pointer.
*
* Needed in some D:YAML code that needs a queue-like structure without too
* much reallocation that goes with an array.
*
* This should be replaced once Phobos has a decent queue/linked list.
*
* Uses manual allocation through malloc/free.
*
* Also has some features uncommon for a queue, e.g. iteration.
* Couldn't bother with implementing a range, as this is used only as
* a placeholder until Phobos gets a decent replacement.
*/
struct Queue(T)
{
private:
///Linked list node containing one element and pointer to the next node.
struct Node
{
T payload_ = T.init;
Node* next_ = null;
}
///Start of the linked list - first element added in time (end of the queue).
Node* first_ = null;
///Last element of the linked list - last element added in time (start of the queue).
Node* last_ = null;
///Cursor pointing to the current node in iteration.
Node* cursor_ = null;
///Length of the queue.
size_t length_ = 0;
public:
@disable void opAssign(ref Queue);
///Destroy the queue, deallocating all its elements.
~this()
{
while(!empty){pop();}
cursor_ = last_ = first_ = null;
length_ = 0;
}
///Start iterating over the queue.
void startIteration()
{
cursor_ = first_;
}
///Get next element in the queue.
ref const(T) next()
in
{
assert(!empty);
assert(cursor_ !is null);
}
body
{
const previous = cursor_;
cursor_ = cursor_.next_;
return previous.payload_;
}
///Are we done iterating?
bool iterationOver() const
{
return cursor_ is null;
}
///Push new item to the queue.
void push(in T item)
{
Node* newLast = allocate!Node(item, cast(Node*)null);
if(last_ !is null){last_.next_ = newLast;}
if(first_ is null){first_ = newLast;}
last_ = newLast;
++length_;
}
///Insert a new item putting it to specified index in the linked list.
void insert(in T item, in size_t idx)
in
{
assert(idx <= length_);
}
body
{
if(idx == 0)
{
//Add after the first element - so this will be the next to pop.
first_ = allocate!Node(item, first_);
++length_;
}
else if(idx == length_)
{
//Adding before last added element, so we can just push.
push(item);
}
else
{
//Get the element before one we're inserting.
Node* current = first_;
foreach(i; 1 .. idx)
{
current = current.next_;
}
//Insert a new node after current, and put current.next_ behind it.
current.next_ = allocate!Node(item, current.next_);
++length_;
}
}
///Return the next element in the queue and remove it.
T pop()
in
{
assert(!empty, "Trying to pop an element from an empty queue");
}
body
{
T result = peek();
Node* temp = first_;
first_ = first_.next_;
free(temp);
if(--length_ == 0)
{
assert(first_ is null);
last_ = null;
}
return result;
}
///Return the next element in the queue.
ref const(T) peek() const
in
{
assert(!empty, "Trying to peek at an element in an empty queue");
}
body
{
return first_.payload_;
}
///Is the queue empty?
@property bool empty() const
{
return first_ is null;
}
///Return number of elements in the queue.
@property size_t length() const
{
return length_;
}
}
private:
///Allocate a struct, passing arguments to its constructor or default initializer.
T* allocate(T, Args...)(Args args)
{
T* ptr = cast(T*)malloc(T.sizeof);
*ptr = T(args);
//The struct might contain references to GC-allocated memory, so tell the GC about it.
GC.addRange(cast(void*)ptr, T.sizeof);
return ptr;
}
///Deallocate struct pointed at by specified pointer.
void free(T)(T* ptr)
{
//GC doesn't need to care about any references in this struct anymore.
GC.removeRange(cast(void*)ptr);
clear(*ptr);
std.c.stdlib.free(ptr);
}
unittest
{
auto queue = Queue!int();
assert(queue.empty);
foreach(i; 0 .. 65)
{
queue.push(5);
assert(queue.pop() == 5);
assert(queue.empty);
assert(queue.length_ == 0);
}
int[] array = [1, -1, 2, -2, 3, -3, 4, -4, 5, -5];
foreach(i; array)
{
queue.push(i);
}
array = 42 ~ array[0 .. 3] ~ 42 ~ array[3 .. $] ~ 42;
queue.insert(42, 3);
queue.insert(42, 0);
queue.insert(42, queue.length);
int[] array2;
while(!queue.empty)
{
array2 ~= queue.pop();
}
assert(array == array2);
}

58
dyaml/scanner.d
View file

@ -23,6 +23,7 @@ import std.typecons;
import std.utf;
import dyaml.exception;
import dyaml.queue;
import dyaml.reader;
import dyaml.token;
@ -119,9 +120,9 @@ final class Scanner
///Past indentation levels. Used as a stack.
int[] indents_;
//Should be replaced by a queue or linked list once Phobos has anything usable.
///Processed tokens not yet emitted. Used as a queue.
Token[] tokens_;
Queue!Token tokens_;
///Number of tokens emitted through the getToken method.
uint tokensTaken_;
@ -152,7 +153,6 @@ final class Scanner
~this()
{
clear(tokens_);
tokens_ = null;
clear(indents_);
indents_ = null;
clear(possibleSimpleKeys_);
@ -180,7 +180,7 @@ final class Scanner
if(ids.length == 0){return true;}
else
{
const nextId = tokens_.front.id;
const nextId = tokens_.peek().id;
foreach(id; ids)
{
if(nextId == id){return true;}
@ -195,10 +195,10 @@ final class Scanner
*
* Must not be called if there are no tokens left.
*/
ref Token peekToken()
ref const(Token) peekToken()
{
while(needMoreTokens){fetchToken();}
if(!tokens_.empty){return tokens_.front;}
if(!tokens_.empty){return tokens_.peek();}
assert(false, "No token left to peek");
}
@ -213,9 +213,7 @@ final class Scanner
if(!tokens_.empty)
{
++tokensTaken_;
Token result = tokens_.front;
tokens_.popFront();
return result;
return tokens_.pop();
}
assert(false, "No token left to get");
}
@ -380,7 +378,7 @@ final class Scanner
{
indent_ = indents_.back;
indents_.popBack();
tokens_ ~= blockEndToken(reader_.mark, reader_.mark);
tokens_.push(blockEndToken(reader_.mark, reader_.mark));
}
}
@ -403,7 +401,7 @@ final class Scanner
///Add STREAM-START token.
void fetchStreamStart()
{
tokens_ ~= streamStartToken(reader_.mark, reader_.mark, reader_.encoding);
tokens_.push(streamStartToken(reader_.mark, reader_.mark, reader_.encoding));
}
///Add STREAM-END token.
@ -417,7 +415,7 @@ final class Scanner
SimpleKey[uint] empty;
possibleSimpleKeys_ = empty;
tokens_ ~= streamEndToken(reader_.mark, reader_.mark);
tokens_.push(streamEndToken(reader_.mark, reader_.mark));
done_ = true;
}
@ -430,7 +428,7 @@ final class Scanner
removePossibleSimpleKey();
allowSimpleKey_ = false;
tokens_ ~= scanDirective();
tokens_.push(scanDirective());
}
///Add DOCUMENT-START or DOCUMENT-END token.
@ -445,7 +443,7 @@ final class Scanner
Mark startMark = reader_.mark;
reader_.forward(3);
tokens_ ~= simpleToken!id(startMark, reader_.mark);
tokens_.push(simpleToken!id(startMark, reader_.mark));
}
///Aliases to add DOCUMENT-START or DOCUMENT-END token.
@ -463,7 +461,7 @@ final class Scanner
Mark startMark = reader_.mark;
reader_.forward();
tokens_ ~= simpleToken!id(startMark, reader_.mark);
tokens_.push(simpleToken!id(startMark, reader_.mark));
}
///Aliases to add FLOW-SEQUENCE-START or FLOW-MAPPING-START token.
@ -481,7 +479,7 @@ final class Scanner
Mark startMark = reader_.mark;
reader_.forward();
tokens_ ~= simpleToken!id(startMark, reader_.mark);
tokens_.push(simpleToken!id(startMark, reader_.mark));
}
///Aliases to add FLOW-SEQUENCE-START or FLOW-MAPPING-START token/
@ -498,7 +496,7 @@ final class Scanner
Mark startMark = reader_.mark;
reader_.forward();
tokens_ ~= flowEntryToken(startMark, reader_.mark);
tokens_.push(flowEntryToken(startMark, reader_.mark));
}
/**
@ -515,7 +513,7 @@ final class Scanner
if(addIndent(reader_.column))
{
tokens_ ~= simpleToken!id(reader_.mark, reader_.mark);
tokens_.push(simpleToken!id(reader_.mark, reader_.mark));
}
}
@ -534,7 +532,7 @@ final class Scanner
Mark startMark = reader_.mark;
reader_.forward();
tokens_ ~= blockEntryToken(startMark, reader_.mark);
tokens_.push(blockEntryToken(startMark, reader_.mark));
}
///Add KEY token. Might add BLOCK-MAPPING-START in the process.
@ -549,7 +547,7 @@ final class Scanner
Mark startMark = reader_.mark;
reader_.forward();
tokens_ ~= keyToken(startMark, reader_.mark);
tokens_.push(keyToken(startMark, reader_.mark));
}
///Add VALUE token. Might add KEY and/or BLOCK-MAPPING-START in the process.
@ -567,14 +565,12 @@ final class Scanner
//Add KEY.
//Manually inserting since tokens are immutable (need linked list).
tokens_ = tokens_[0 .. idx] ~ keyToken(keyMark, keyMark) ~
tokens_[idx .. tokens_.length];
tokens_.insert(keyToken(keyMark, keyMark), idx);
//If this key starts a new block mapping, we need to add BLOCK-MAPPING-START.
if(flowLevel_ == 0 && addIndent(key.column))
{
tokens_ = tokens_[0 .. idx] ~ blockMappingStartToken(keyMark, keyMark) ~
tokens_[idx .. tokens_.length];
tokens_.insert(blockMappingStartToken(keyMark, keyMark), idx);
}
//There cannot be two simple keys in a row.
@ -591,7 +587,7 @@ final class Scanner
//BLOCK-MAPPING-START. It'll be detected as an error later by the parser.
if(flowLevel_ == 0 && addIndent(reader_.column))
{
tokens_ ~= blockMappingStartToken(reader_.mark, reader_.mark);
tokens_.push(blockMappingStartToken(reader_.mark, reader_.mark));
}
//Reset possible simple key on the current level.
@ -603,7 +599,7 @@ final class Scanner
//Add VALUE.
Mark startMark = reader_.mark;
reader_.forward();
tokens_ ~= valueToken(startMark, reader_.mark);
tokens_.push(valueToken(startMark, reader_.mark));
}
///Add ALIAS or ANCHOR token.
@ -615,7 +611,7 @@ final class Scanner
//No simple keys after ALIAS/ANCHOR.
allowSimpleKey_ = false;
tokens_ ~= scanAnchor(id);
tokens_.push(scanAnchor(id));
}
///Aliases to add ALIAS or ANCHOR token.
@ -630,7 +626,7 @@ final class Scanner
//No simple keys after TAG.
allowSimpleKey_ = false;
tokens_ ~= scanTag();
tokens_.push(scanTag());
}
///Add block SCALAR token.
@ -642,7 +638,7 @@ final class Scanner
//A simple key may follow a block scalar.
allowSimpleKey_ = true;
tokens_ ~= scanBlockScalar(style);
tokens_.push(scanBlockScalar(style));
}
///Aliases to add literal or folded block scalar.
@ -658,7 +654,7 @@ final class Scanner
allowSimpleKey_ = false;
//Scan and add SCALAR.
tokens_ ~= scanFlowScalar(quotes);
tokens_.push(scanFlowScalar(quotes));
}
///Aliases to add single or double quoted block scalar.
@ -675,7 +671,7 @@ final class Scanner
allowSimpleKey_ = false;
//Scan and add SCALAR. May change allowSimpleKey_
tokens_ ~= scanPlain();
tokens_.push(scanPlain());
}

2
dyaml/token.d
View file

@ -69,7 +69,7 @@ enum CollectionStyle : ubyte
*
* 32 bytes on 64-bit.
*/
immutable struct Token
struct Token
{
///Value of the token, if any.
string value;