From f183e4aa800a4b7a0c4a994d972256715b8484ac Mon Sep 17 00:00:00 2001 From: Cameron Ross Date: Tue, 8 Jan 2019 18:42:39 -0330 Subject: [PATCH] add an official yaml test suite runner (#209) add an official yaml test suite runner merged-on-behalf-of: BBasile --- .travis.yml | 3 +- dub.json | 3 +- testsuite/dub.json | 8 + testsuite/source/app.d | 323 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 335 insertions(+), 2 deletions(-) create mode 100644 testsuite/dub.json create mode 100644 testsuite/source/app.d diff --git a/.travis.yml b/.travis.yml index 3388ee4..7e284b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ os: d: - dmd - ldc - + branches: only: - master @@ -32,6 +32,7 @@ script: - "dub build dyaml:getting-started" - "dub build dyaml:representer" - "dub build dyaml:resolver" + - "dub build dyaml:testsuite" - "dub build dyaml:tojson" - "dub build dyaml:yaml_gen" - "dub build dyaml:yaml_stats" diff --git a/dub.json b/dub.json index 17449ec..07ee17e 100644 --- a/dub.json +++ b/dub.json @@ -19,6 +19,7 @@ "examples/tojson", "examples/yaml_bench", "examples/yaml_gen", - "examples/yaml_stats" + "examples/yaml_stats", + "testsuite" ] } diff --git a/testsuite/dub.json b/testsuite/dub.json new file mode 100644 index 0000000..fece639 --- /dev/null +++ b/testsuite/dub.json @@ -0,0 +1,8 @@ +{ + "name": "testsuite", + "targetType": "executable", + "dependencies": + { + "dyaml": "*" + } +} diff --git a/testsuite/source/app.d b/testsuite/source/app.d new file mode 100644 index 0000000..13c79a0 --- /dev/null +++ b/testsuite/source/app.d @@ -0,0 +1,323 @@ +module dyaml.testsuite; + +import dyaml; +import dyaml.event; + +import std.algorithm; +import std.conv; +import std.file; +import std.format; +import std.json; +import std.path; +import std.range; +import std.stdio; +import std.string; +import std.typecons; +import std.utf; + +auto dumpEventString(string str) @safe +{ + string[] output; + try + { + auto events = Loader.fromString(str).parse(); + foreach (event; events) + { + string line; + final switch (event.id) + { + case EventID.scalar: + line = "=VAL "; + if (event.anchor != "") + { + line ~= text("&", event.anchor, " "); + } + if (event.tag != "") + { + line ~= text("<", event.tag, "> "); + } + switch(event.scalarStyle) + { + case ScalarStyle.singleQuoted: + line ~= "'"; + break; + case ScalarStyle.doubleQuoted: + line ~= '"'; + break; + case ScalarStyle.literal: + line ~= "|"; + break; + case ScalarStyle.folded: + line ~= ">"; + break; + default: + line ~= ":"; + break; + } + if (event.value != "") + { + line ~= text(event.value.substitute("\n", "\\n", `\`, `\\`, "\r", "\\r", "\t", "\\t", "\b", "\\b")); + } + break; + case EventID.streamStart: + line = "+STR"; + break; + case EventID.documentStart: + line = "+DOC"; + if (event.explicitDocument) + { + line ~= text(" ---"); + } + break; + case EventID.mappingStart: + line = "+MAP"; + if (event.anchor != "") + { + line ~= text(" &", event.anchor); + } + if (event.tag != "") + { + line ~= text(" <", event.tag, ">"); + } + break; + case EventID.sequenceStart: + line = "+SEQ"; + if (event.anchor != "") + { + line ~= text(" &", event.anchor); + } + if (event.tag != "") + { + line ~= text(" <", event.tag, ">"); + } + break; + case EventID.streamEnd: + line = "-STR"; + break; + case EventID.documentEnd: + line = "-DOC"; + if (event.explicitDocument) + { + line ~= " ..."; + } + break; + case EventID.mappingEnd: + line = "-MAP"; + break; + case EventID.sequenceEnd: + line = "-SEQ"; + break; + case EventID.alias_: + line = text("=ALI *", event.anchor); + break; + case EventID.invalid: + assert(0, "Invalid EventID produced"); + } + output ~= line; + } + } + catch (Exception) {} //Exceptions should just stop adding output + return output.join("\n"); +} + +enum TestState +{ + success, + skipped, + failure +} + +struct TestResult +{ + string name; + TestState state; + string failMsg; + + const void toString(OutputRange)(ref OutputRange writer) + if (isOutputRange!(OutputRange, char)) + { + ubyte statusColour; + string statusString; + final switch (state) { + case TestState.success: + statusColour = 32; + statusString = "Succeeded"; + break; + case TestState.failure: + statusColour = 31; + statusString = "Failed"; + break; + case TestState.skipped: + statusColour = 93; + statusString = "Skipped"; + break; + } + writer.formattedWrite!"[\033[%s;1m%s\033[0m] %s"(statusColour, statusString, name); + if (state != TestState.success) + { + writer.formattedWrite!" (%s)"(failMsg.replace("\n", " ")); + } + } +} + +TestResult runTests(string tml) @safe +{ + TestResult output; + output.state = TestState.success; + auto splitFile = tml.splitter("\n--- "); + output.name = splitFile.front.findSplit("=== ")[2]; + bool loadFailed, shouldFail; + string failMsg; + JSONValue json; + Node[] nodes; + string yamlString; + Nullable!string compareYAMLString; + Nullable!string events; + ulong testsRun; + + void fail(string msg) @safe + { + output.state = TestState.failure; + output.failMsg = msg; + } + void skip(string msg) @safe + { + output.state = TestState.skipped; + output.failMsg = msg; + } + void parseYAML(string yaml) @safe + { + yamlString = yaml; + try { + nodes = Loader.fromString(yamlString).array; + } + catch (Exception e) + { + loadFailed = true; + failMsg = e.msg; + } + } + void compareLineByLine(const string a, const string b, const string msg) @safe + { + foreach (line1, line2; zip(a.lineSplitter, b.lineSplitter)) + { + if (line1 != line2) + { + fail(text(msg, " Got ", line1, ", expected ", line2)); + break; + } + } + } + foreach (section; splitFile.drop(1)) + { + auto splitSection = section.findSplit("\n"); + auto splitSectionHeader = splitSection[0].findSplit(":"); + const splitSectionName = splitSectionHeader[0].findSplit("("); + const sectionName = splitSectionName[0]; + const sectionParams = splitSectionName[2].findSplit(")")[0]; + string sectionData = splitSection[2]; + if (sectionData != "") + { + //< means dedent. + if (sectionParams.canFind("<")) + { + sectionData = sectionData[4..$].substitute("\n ", "\n", "", " ", "", "\t").toUTF8; + } + else + { + sectionData = sectionData.substitute("", " ", "", "\t").toUTF8; + } + //Not sure what + means. + } + switch(sectionName) + { + case "in-yaml": + parseYAML(sectionData); + break; + case "in-json": + json = parseJSON(sectionData); + break; + case "test-event": + events = sectionData; + break; + case "error": + shouldFail = true; + testsRun++; + break; + case "out-yaml": + compareYAMLString = sectionData; + break; + case "emit-yaml": + // TODO: Figure out how/if to implement this + //fail("Unhandled test - emit-yaml"); + break; + case "lex-token": + // TODO: Should this be implemented? + //fail("Unhandled test - lex-token"); + break; + case "from": break; + case "tags": break; + default: assert(false, text("Unhandled section ", sectionName, "in ", output.name)); + } + } + if (!loadFailed && !compareYAMLString.isNull && !shouldFail) + { + Appender!string buf; + dumper(buf).dump(); + compareLineByLine(buf.data, compareYAMLString, "Dumped YAML mismatch"); + testsRun++; + } + if (!loadFailed && !events.isNull && !shouldFail) + { + const compare = dumpEventString(yamlString); + compareLineByLine(compare, events, "Event mismatch"); + testsRun++; + } + if (loadFailed && !shouldFail) + { + fail(failMsg); + } + if (shouldFail && !loadFailed) + { + fail("Invalid YAML accepted"); + } + if ((testsRun == 0) && (output.state != TestState.failure)) + { + skip("No tests run"); + } + return output; +} + +// Can't be @safe due to dirEntries() +void main(string[] args) @system +{ + string path = "yaml-test-suite/test"; + + void printResult(string id, TestResult result) + { + writeln(id, " ", result); + } + + if (args.length > 1) + { + path = args[1]; + } + + ulong total; + ulong successes; + foreach (file; dirEntries(path, "*.tml", SpanMode.shallow)) + { + auto result = runTests(readText(file)); + if (result.state == TestState.success) + { + debug(verbose) printResult(file.baseName, result); + successes++; + } + else + { + printResult(file.baseName, result); + } + total++; + } + writefln!"%d/%d tests passed"(successes, total); +}