From 45738dd02096bf5cbf3042f571ae696526324383 Mon Sep 17 00:00:00 2001 From: Javier Maestro Date: Thu, 16 May 2024 20:57:29 +0100 Subject: [PATCH] Improve testing with stats and errors per test section * TestRunner: separated the running of each "section" of a test into its own try-catch so that we can evaluate each section separately even if another section failed (e.g. evaluate all examples even if one test in "facts" throws an exception). I've added tests that exercise this "new feature". * TestResults: refactored it a bit so that TestResults now has the three sections and the old TestResults becomes TestSectionResults. Also, TestSectionResults now has only *one* error, since that's what we get when there's an exception evaluating the Pkl code. * SimpleReport: * Separated reporting of each test section so that it's easy to see which facts or which examples fail. * Added a "stats line" to the module level that reports how many tests and assertions pass/fail. Note that examples are equivalent to "one test with an 'all-or-nothing assertion' so they are counted as 1 test with 1 assertion. Similarly, errors will also count as "one test with one assertion failed" because an error prevents any tests in a section from being evaluated (that is, "the whole section fails" until the error is corrected and we can then evaluate the assertions in the tests). * For the examples converted to tests, when multiple examples share the same name I've also added a counter ("# 1" and so on) to disambiguate them so that it's easier to identify which one is failing. * JUnitReport: fixed the reporting of failures as it wasn't consistent with "tests": the tests were the number of tests that we run but "failures" were the number of assertions that failed. --- .../kotlin/org/pkl/cli/CliTestRunnerTest.kt | 24 +- .../org/pkl/core/runtime/TestResults.java | 407 +++++++++++------- .../java/org/pkl/core/runtime/TestRunner.java | 50 ++- .../core/stdlib/test/report/JUnitReport.java | 76 ++-- .../core/stdlib/test/report/SimpleReport.java | 97 +++-- .../kotlin/org/pkl/core/EvaluateTestsTest.kt | 137 +++++- .../test/kotlin/org/pkl/gradle/TestsTest.kt | 346 ++++++++++----- 7 files changed, 790 insertions(+), 347 deletions(-) diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt index 79435e939..6b8eb8062 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt @@ -63,8 +63,11 @@ class CliTestRunnerTest { .isEqualTo( """ module test - succeed ✅ + facts + ✅ succeed + ✅ 100.0% tests pass [1 passed], 100.0% asserts pass [2 passed] + """ .trimIndent() ) @@ -80,7 +83,7 @@ class CliTestRunnerTest { facts { ["fail"] { 4 == 9 - "foo" == "bar" + "foo" != "bar" } } """ @@ -97,10 +100,12 @@ class CliTestRunnerTest { .isEqualTo( """ module test - fail ❌ - 4 == 9 ❌ - "foo" == "bar" ❌ + facts + ❌ fail + 4 == 9 + ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed] + """ .trimIndent() ) @@ -118,7 +123,8 @@ class CliTestRunnerTest { 9 == trace(9) "foo" == "foo" } - ["fail"] { + ["bar"] { + "foo" == "foo" 5 == 9 } } @@ -136,9 +142,9 @@ class CliTestRunnerTest { """ - - - 5 == 9 ❌ + + + 5 == 9 diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java index 9b4c149de..52fcb6c91 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java @@ -20,61 +20,40 @@ import java.util.Collections; import java.util.List; import org.pkl.core.PklException; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestSection; /** Aggregate test results for a module. Used to verify test failures and generate reports. */ public final class TestResults { - - private final String module; - private final String displayUri; - private final List results = new ArrayList<>(); + public final String moduleName; + public final String displayUri; + public final TestSectionResults module = new TestSectionResults(TestSection.MODULE); + public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS); + public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES); private String err = ""; - public TestResults(String module, String displayUri) { - this.module = module; + public TestResults(String moduleName, String displayUri) { + this.moduleName = moduleName; this.displayUri = displayUri; } - public String getModuleName() { - return module; - } - - public String getDisplayUri() { - return displayUri; - } - - public List getResults() { - return Collections.unmodifiableList(results); - } - - public TestResult newResult(String name) { - var result = new TestResult(name); - results.add(result); - return result; + public int totalTests() { + return module.totalTests() + facts.totalTests() + examples.totalTests(); } - public void newResult(String name, Failure failure) { - var result = new TestResult(name); - result.addFailure(failure); - results.add(result); + public int totalFailures() { + return module.totalFailures() + facts.totalFailures() + examples.totalFailures(); } - public int totalTests() { - return results.size(); + public int totalAsserts() { + return module.totalAsserts() + facts.totalAsserts() + examples.totalAsserts(); } - public int totalFailures() { - int total = 0; - for (var res : results) { - total += res.getFailures().size(); - } - return total; + public int totalAssertsFailed() { + return module.totalAssertsFailed() + facts.totalAssertsFailed() + examples.totalAssertsFailed(); } public boolean failed() { - for (var res : results) { - if (res.isFailure()) return true; - } - return false; + return module.failed() || facts.failed() || examples.failed(); } public String getErr() { @@ -85,147 +64,277 @@ public void setErr(String err) { this.err = err; } - public static class TestResult { + public static class TestSectionResults { + public final TestSection name; + private final List results = new ArrayList<>(); + private Error error; - private final String name; - private final List failures = new ArrayList<>(); - private final List errors = new ArrayList<>(); - private boolean isExampleWritten = false; - - public TestResult(String name) { + public TestSectionResults(TestSection name) { this.name = name; } - public boolean isSuccess() { - return failures.isEmpty() && errors.isEmpty(); + public void setError(Error error) { + this.error = error; } - boolean isFailure() { - return !isSuccess(); + public Error getError() { + return error; } - public String getName() { - return name; + public boolean hasError() { + return error != null; } - public boolean isExampleWritten() { - return isExampleWritten; + public List getResults() { + return Collections.unmodifiableList(results); } - public void setExampleWritten(boolean exampleWritten) { - isExampleWritten = exampleWritten; + public TestResult newResult(String name) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + results.add(result); + return result; } - public List getFailures() { - return Collections.unmodifiableList(failures); + public void newResult(String name, Failure failure) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + result.addFailure(failure); + results.add(result); } - public void addFailure(Failure description) { - failures.add(description); + public int totalTests() { + var total = results.size(); + return (hasError() ? ++total : total); } - public List getErrors() { - return Collections.unmodifiableList(errors); + public int totalAsserts() { + int total = 0; + for (var res : results) { + total += res.totalAsserts(); + } + return (hasError() ? ++total : total); } - public void addError(Error err) { - errors.add(err); + public int totalAssertsFailed() { + int total = 0; + for (var res : results) { + total += res.totalAssertsFailed(); + } + return (hasError() ? ++total : total); } - } - public static class Failure { - - private final String kind; - private final String rendered; - - private Failure(String kind, String rendered) { - this.kind = kind; - this.rendered = rendered; - } - - public String getKind() { - return kind; - } - - public String getRendered() { - return rendered; - } - - public static Failure buildFactFailure(SourceSection sourceSection, String description) { - return new Failure( - "Fact Failure", sourceSection.getCharacters() + " ❌ (" + description + ")"); - } - - public static Failure buildExampleLengthMismatchFailure( - String location, String property, int expectedLength, int actualLength) { - String builder = - "(" - + location - + ")\n" - + "Output mismatch: Expected \"" - + property - + "\" to contain " - + expectedLength - + " examples, but found " - + actualLength; - return new Failure("Output Mismatch (Length)", builder); - } - - public static Failure buildExamplePropertyMismatchFailure( - String location, String property, boolean isMissingInExpected) { - var builder = new StringBuilder(); - builder - .append("(") - .append(location) - .append(")\n") - .append("Output mismatch: \"") - .append(property); - if (isMissingInExpected) { - builder.append("\" exists in actual but not in expected output"); - } else { - builder.append("\" exists in expected but not in actual output"); - } - return new Failure("Output Mismatch", builder.toString()); - } - - public static Failure buildExampleFailure( - String location, - String expectedLocation, - String expectedValue, - String actualLocation, - String actualValue) { - String builder = - "(" - + location - + ")\n" - + "Expected: (" - + expectedLocation - + ")\n" - + expectedValue - + "\nActual: (" - + actualLocation - + ")\n" - + actualValue; - return new Failure("Example Failure", builder); + public int totalFailures() { + int total = 0; + for (var res : results) { + if (res.isFailure()) total++; + } + return (hasError() ? ++total : total); } - } - public static class Error { + public boolean failed() { + if (hasError()) return true; + + for (var res : results) { + if (res.isFailure()) return true; + } + return false; + } + + public static class TestResult { + public final String name; + private int totalAsserts = 0; + private int totalAssertsFailed = 0; + private final List failures = new ArrayList<>(); + public final boolean isExample; + private boolean isExampleWritten = false; + + public TestResult(String name, boolean isExample) { + this.name = name; + this.isExample = isExample; + } + + public boolean isSuccess() { + return failures.isEmpty(); + } + + public boolean isFailure() { + return !isSuccess(); + } + + public boolean isExampleWritten() { + return isExampleWritten; + } - private final String message; - private final PklException exception; + public void setExampleWritten(boolean exampleWritten) { + isExampleWritten = exampleWritten; + } + + public int totalAsserts() { + return totalAsserts; + } + + public void countAssert() { + totalAsserts++; + } - public Error(String message, PklException exception) { - this.message = message; - this.exception = exception; + public List getFailures() { + return Collections.unmodifiableList(failures); + } + + public int totalAssertsFailed() { + return totalAssertsFailed; + } + + public void addFailure(Failure description) { + failures.add(description); + totalAssertsFailed++; + } } - public String getMessage() { - return message; + public static class Failure { + + private final String kind; + private final String failure; + private final String location; + + private Failure(String kind, String failure, String location) { + this.kind = kind; + this.failure = failure; + this.location = location; + } + + public String getKind() { + return kind; + } + + public String getFailure() { + return failure; + } + + public String getLocation() { + return location; + } + + public static String renderLocation(String location) { + return "(" + location + ")"; + } + + public String getRendered() { + String rendered; + + if (kind == "Fact Failure") { + rendered = failure + " " + renderLocation(getLocation()); + } else { + rendered = renderLocation(getLocation()) + "\n" + failure; + } + + return rendered; + } + + public static Failure buildFactFailure(String location, SourceSection sourceSection) { + return new Failure("Fact Failure", sourceSection.getCharacters().toString(), location); + } + + public static Failure buildExampleLengthMismatchFailure( + String location, String property, int expectedLength, int actualLength) { + var builder = new StringBuilder(); + builder + .append("Output mismatch: Expected \"") + .append(property) + .append("\" to contain ") + .append(expectedLength) + .append(" examples, but found ") + .append(actualLength); + + return new Failure("Output Mismatch (Length)", builder.toString(), location); + } + + public static Failure buildExamplePropertyMismatchFailure( + String location, String property, boolean isMissingInExpected) { + + String exists_in; + String missing_in; + + if (isMissingInExpected) { + exists_in = "actual"; + missing_in = "expected"; + } else { + exists_in = "expected"; + missing_in = "actual"; + } + + var builder = new StringBuilder(); + builder + .append("Output mismatch: \"") + .append(property) + .append("\" exists in ") + .append(exists_in) + .append(" but not in ") + .append(missing_in) + .append(" output"); + + return new Failure("Output Mismatch", builder.toString(), location); + } + + public static Failure buildExampleFailure( + String location, + String expectedLocation, + String expectedValue, + String actualLocation, + String actualValue) { + var builder = new StringBuilder(); + builder + .append("Expected: ") + .append(renderLocation(expectedLocation)) + .append("\n") + .append(expectedValue) + .append("\n") + .append("Actual: ") + .append(renderLocation(actualLocation)) + .append("\n") + .append(actualValue); + + return new Failure("Example Failure", builder.toString(), location); + } } - public Exception getException() { - return exception; + public static class Error { + + private final String message; + private final PklException exception; + + public Error(String message, PklException exception) { + this.message = message; + this.exception = exception; + } + + public String getMessage() { + return message; + } + + public Exception getException() { + return exception; + } + + public String getRendered() { + return exception.getMessage(); + } + } + + public enum TestSection { + MODULE("module"), + FACTS("facts"), + EXAMPLES("examples"); + + private final String name; + + TestSection(final String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } } } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java index 47da5a83e..67f3658d2 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java @@ -23,8 +23,9 @@ import org.pkl.core.StackFrameTransformer; import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.module.ModuleKeys; -import org.pkl.core.runtime.TestResults.Error; -import org.pkl.core.runtime.TestResults.Failure; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.Failure; import org.pkl.core.stdlib.PklConverter; import org.pkl.core.stdlib.base.PcfRenderer; import org.pkl.core.util.EconomicMaps; @@ -51,12 +52,25 @@ public TestResults run(VmTyped testModule) { try { checkAmendsPklTest(testModule); - runFacts(testModule, results); - runExamples(testModule, info, results); } catch (VmException v) { - var meta = results.newResult(info.getModuleName()); - meta.addError(new Error(v.getMessage(), v.toPklException(stackFrameTransformer))); + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.module.setError(error); } + + try { + runFacts(testModule, results.facts); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.facts.setError(error); + } + + try { + runExamples(testModule, info, results.examples); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.examples.setError(error); + } + results.setErr(logger.getLogs()); return results; } @@ -72,7 +86,7 @@ private void checkAmendsPklTest(VmTyped value) { } } - private void runFacts(VmTyped testModule, TestResults results) { + private void runFacts(VmTyped testModule, TestSectionResults results) { var facts = VmUtils.readMember(testModule, Identifier.FACTS); if (facts instanceof VmNull) return; @@ -83,11 +97,13 @@ private void runFacts(VmTyped testModule, TestResults results) { var groupListing = (VmListing) groupValue; groupListing.forceAndIterateMemberValues( ((factIndex, factMember, factValue) -> { + result.countAssert(); + assert factValue instanceof Boolean; if (factValue == Boolean.FALSE) { result.addFailure( Failure.buildFactFailure( - factMember.getSourceSection(), getDisplayUri(factMember))); + getDisplayUri(factMember), factMember.getSourceSection())); } return true; })); @@ -95,7 +111,7 @@ private void runFacts(VmTyped testModule, TestResults results) { }); } - private void runExamples(VmTyped testModule, ModuleInfo info, TestResults results) { + private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) { var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES); if (examples instanceof VmNull) return; @@ -138,7 +154,10 @@ private void runExamples(VmTyped testModule, ModuleInfo info, TestResults result } private void doRunAndValidateExamples( - VmMapping examples, Path expectedOutputFile, Path actualOutputFile, TestResults results) { + VmMapping examples, + Path expectedOutputFile, + Path actualOutputFile, + TestSectionResults results) { var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile); var actualExampleOutputs = new MutableReference(null); var allGroupsSucceeded = new MutableBoolean(true); @@ -202,8 +221,11 @@ private void doRunAndValidateExamples( .build(); } + var exampleName = + group.getLength() == 1 ? testName : testName + " #" + exampleIndex; + results.newResult( - testName, + exampleName, Failure.buildExampleFailure( getDisplayUri(exampleMember), getDisplayUri(expectedMember), @@ -244,10 +266,12 @@ private void doRunAndValidateExamples( } } - private void doRunAndWriteExamples(VmMapping examples, Path outputFile, TestResults results) { + private void doRunAndWriteExamples( + VmMapping examples, Path outputFile, TestSectionResults results) { examples.forceAndIterateMemberValues( (groupKey, groupMember, groupValue) -> { - results.newResult(String.valueOf(groupKey)).setExampleWritten(true); + var example = results.newResult(String.valueOf(groupKey)); + example.setExampleWritten(true); return true; }); writeExampleOutputs(outputFile, examples); diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java index c5c10eeff..ca8378c72 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java @@ -24,7 +24,9 @@ import org.pkl.core.runtime.BaseModule; import org.pkl.core.runtime.Identifier; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.runtime.VmDynamic; import org.pkl.core.runtime.VmMapping; import org.pkl.core.runtime.VmTyped; @@ -41,27 +43,46 @@ public void report(TestResults results, Writer writer) throws IOException { writer.append(renderXML(" ", "1.0", buildSuite(results))); } - private VmDynamic buildSuite(TestResults res) { - var testCases = testCases(res); - if (!res.getErr().isBlank()) { + private VmDynamic buildSuite(TestResults results) { + var testCases = testCases(results.moduleName, results.facts); + testCases.addAll(testCases(results.moduleName, results.examples)); + + if (!results.getErr().isBlank()) { var err = buildXmlElement( "system-err", VmMapping.empty(), - members -> members.put("body", syntheticElement(makeCdata(res.getErr())))); + members -> members.put("body", syntheticElement(makeCdata(results.getErr())))); testCases.add(err); } - return buildXmlElement( - "testsuite", buildRootAttributes(res), testCases.toArray(new VmDynamic[0])); + + var attrs = + buildAttributes( + "name", results.moduleName, + "tests", (long) results.totalTests(), + "failures", (long) results.totalFailures()); + + return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0])); } - private ArrayList testCases(TestResults results) { - var className = results.getModuleName(); - var elements = new ArrayList(results.totalTests()); - for (var res : results.getResults()) { - var attrs = buildAttributes("classname", className, "name", res.getName()); + private ArrayList testCases(String moduleName, TestSectionResults testSectionResults) { + var elements = new ArrayList(testSectionResults.totalTests()); + + if (testSectionResults.hasError()) { + var error = error(testSectionResults.getError()); + + var attrs = + buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error"); + var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0])); + + elements.add(element); + } + + for (var res : testSectionResults.getResults()) { + var attrs = + buildAttributes( + "classname", moduleName + "." + testSectionResults.name, "name", res.name); var failures = failures(res); - failures.addAll(errors(res)); var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0])); elements.add(element); } @@ -83,19 +104,14 @@ private ArrayList failures(TestResult res) { return list; } - private ArrayList errors(TestResult res) { + private ArrayList error(Error error) { var list = new ArrayList(); - long i = 0; - for (var error : res.getErrors()) { - var attrs = buildAttributes("message", error.getMessage()); - long element = i++; - list.add( - buildXmlElement( - "error", - attrs, - members -> - members.put(element, syntheticElement(error.getException().getMessage())))); - } + var attrs = buildAttributes("message", error.getMessage()); + list.add( + buildXmlElement( + "error", + attrs, + members -> members.put(1, syntheticElement("\n" + error.getRendered())))); return list; } @@ -130,16 +146,6 @@ private VmDynamic buildXmlElement( members.size() - 4); } - private VmMapping buildRootAttributes(TestResults results) { - return buildAttributes( - "name", - results.getModuleName(), - "tests", - (long) results.totalTests(), - "failures", - (long) results.totalFailures()); - } - private VmMapping buildAttributes(Object... attributes) { EconomicMap attrs = EconomicMaps.create(attributes.length); for (int i = 0; i < attributes.length; i += 2) { diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java index 82e788cef..ce9db655c 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java @@ -19,8 +19,8 @@ import java.io.Writer; import java.util.stream.Collectors; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.Failure; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.util.StringUtils; public final class SimpleReport implements TestReport { @@ -28,38 +28,66 @@ public final class SimpleReport implements TestReport { @Override public void report(TestResults results, Writer writer) throws IOException { var builder = new StringBuilder(); - builder.append("module "); - builder.append(results.getModuleName()); - builder.append(" (").append(results.getDisplayUri()).append(")\n"); - StringUtils.joinToStringBuilder( - builder, results.getResults(), "\n", res -> reportResult(res, builder)); - builder.append("\n"); + + builder.append("module ").append(results.moduleName).append("\n"); + + reportResults(results.facts, builder); + reportResults(results.examples, builder); + + builder.append(results.failed() ? "❌ " : "✅ "); + + var totalStatsLine = + makeStatsLine("tests", results.totalTests(), results.totalFailures(), results.failed()); + builder.append(totalStatsLine); + + var totalAssertsStatsLine = + makeStatsLine( + "asserts", results.totalAsserts(), results.totalAssertsFailed(), results.failed()); + builder.append(", ").append(totalAssertsStatsLine); + + builder.append("\n\n"); + writer.append(builder); } - private void reportResult(TestResult result, StringBuilder builder) { - builder.append(" ").append(result.getName()); - if (result.isExampleWritten()) { - builder.append(" ✍️"); - } else if (result.isSuccess()) { - builder.append(" ✅"); - } else { - builder.append(" ❌\n"); - StringUtils.joinToStringBuilder( - builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder)); + private void reportResults(TestSectionResults section, StringBuilder builder) { + if (!section.getResults().isEmpty()) { + builder.append(" ").append(section.name).append("\n"); + StringUtils.joinToStringBuilder( - builder, - result.getErrors(), - "\n", - error -> { - builder.append(" Error:\n"); - appendPadded(builder, error.getException().getMessage(), " "); - }); + builder, section.getResults(), "\n", res -> reportResult(res, builder)); + builder.append("\n"); + } else if (section.hasError()) { + builder.append(" ").append(section.name).append("\n"); + var error = "Error:\n" + section.getError().getRendered(); + appendPadded(builder, error, " "); + builder.append("\n"); } } - public static void reportFailure(Failure failure, StringBuilder builder) { - appendPadded(builder, failure.getRendered(), " "); + private void reportResult(TestResult result, StringBuilder builder) { + builder.append(" "); + + if (result.isExampleWritten()) { + builder.append(result.name).append(" ✍️"); + } else { + builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name); + + if (!result.isExample) { + var statsLine = + makeStatsLine( + "asserts", result.totalAsserts(), result.getFailures().size(), result.isFailure()); + } + + if (result.isFailure()) { + builder.append("\n"); + StringUtils.joinToStringBuilder( + builder, + result.getFailures(), + "\n", + failure -> appendPadded(builder, failure.getRendered(), " ")); + } + } } private static void appendPadded(StringBuilder builder, String lines, String padding) { @@ -69,4 +97,19 @@ private static void appendPadded(StringBuilder builder, String lines, String pad "\n", str -> builder.append(padding).append(str)); } + + private String makeStatsLine(String kind, int total, int failed, boolean isFailed) { + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0.0; + + String line = String.format("%.1f%% %s pass", pct_passed, kind); + + if (isFailed) { + line += String.format(" [%d/%d failed]", failed, total); + } else { + line += String.format(" [%d passed]", passed); + } + + return line; + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt index 15f99d6b6..bc0c8c5fd 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt @@ -54,7 +54,7 @@ class EvaluateTestsTest { assertThat(results.displayUri).isEqualTo("repl:text") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("should pass") + assertThat(results.facts.results[0].name).isEqualTo("should pass") assertThat(results.err.isBlank()).isTrue } @@ -79,18 +79,19 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] + val res = results.facts.results[0] assertThat(res.name).isEqualTo("should fail") - assertThat(res.errors).isEmpty() + assertThat(results.facts.hasError()).isFalse + assertThat(res.failures.size).isEqualTo(2) val fail1 = res.failures[0] - assertThat(fail1.rendered).isEqualTo("1 == 2 ❌ (repl:text)") + assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)") val fail2 = res.failures[1] - assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" ❌ (repl:text)""") + assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""") } @Test @@ -114,15 +115,14 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(0) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] - assertThat(res.name).isEqualTo("text") - assertThat(res.failures).isEmpty() - assertThat(res.errors.size).isEqualTo(1) + val res = results.facts + assertThat(res.results).isEmpty() + assertThat(res.hasError()).isTrue - val error = res.errors[0] + val error = res.error assertThat(error.message).isEqualTo("got an error") assertThat(error.exception.message) .isEqualTo( @@ -183,7 +183,114 @@ class EvaluateTestsTest { assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("user") + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact failures with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + 1 == 2 + "foo" == "bar" + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results[0].name).isEqualTo("should fail") + assertThat(results.facts.results[0].failures.size).isEqualTo(2) + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact error with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + throw("exception") + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results).isEmpty() + assertThat(results.facts.hasError()).isTrue + assertThat(results.examples.results[0].name).isEqualTo("user") } @Test @@ -228,9 +335,9 @@ class EvaluateTestsTest { assertThat(results.failed()).isTrue assertThat(results.totalFailures()).isEqualTo(1) - val res = results.results[0] + val res = results.examples.results[0] assertThat(res.name).isEqualTo("user") - assertThat(res.errors.isEmpty()).isTrue + assertFalse(results.examples.hasError()) val fail1 = res.failures[0] assertThat(fail1.rendered.stripFileAndLines(tempDir)) diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt index b5a77a954..009a6ee00 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt @@ -30,7 +30,7 @@ class TestsTest : AbstractTest() { writePklFile() val res = runTask("evalTest") - assertThat(res.output).contains("should pass ✅") + assertThat(res.output).contains("✅ should pass") } @Test @@ -49,9 +49,9 @@ class TestsTest : AbstractTest() { ) val res = runTask("evalTest", expectFailure = true) - assertThat(res.output).contains("should fail ❌") - assertThat(res.output).contains("1 == 3 ❌") - assertThat(res.output).contains(""""foo" == "bar" ❌""") + assertThat(res.output).contains("❌ should fail") + assertThat(res.output).contains("1 == 3") + assertThat(res.output).contains(""""foo" == "bar"""") } @Test @@ -68,24 +68,30 @@ class TestsTest : AbstractTest() { .trimIndent() ) - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") - assertThat(output) - .contains( + assertThat(output.trimStart()) + .startsWith( """ - module test (file:///file, line x) - test ❌ + > Task :evalTest FAILED + module test + facts Error: - –– Pkl Error –– - exception - - 9 | throw("exception") - ^^^^^^^^^^^^^^^^^^ - at test#facts["error"][#1] (file:///file, line x) - - 3 | facts { - ^^^^^^^ - at test#facts (file:///file, line x) + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) """ .trimIndent() ) @@ -98,42 +104,52 @@ class TestsTest : AbstractTest() { writeBuildFile() - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") assertThat(output.trimStart()) - .contains( + .startsWith( """ - module test (file:///file, line x) - sum numbers ✅ - divide numbers ✅ - fail ❌ - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) - user 0 ✅ - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Pigeon" - age = 40 - } - Actual: (file:///file, line x) - new { - name = "Pigeon" - age = 41 - } - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Parrot" - age = 35 - } - Actual: (file:///file, line x) - new { - name = "Welma" - age = 35 - } + > Task :evalTest FAILED + pkl: TRACE: 8 = 8 (file:///file, line x) + module test + facts + ✅ sum numbers + ✅ divide numbers + ❌ fail + 4 == 9 (file:///file, line x) + "foo" == "bar" (file:///file, line x) + examples + ✅ user 0 + ❌ user 1 #0 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + ❌ user 1 #1 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + ❌ 50.0% tests pass [3/6 failed] """ .trimIndent() ) @@ -141,28 +157,7 @@ class TestsTest : AbstractTest() { @Test fun `overwrite expected examples`() { - writePklFile( - additionalExamples = - """ - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } - """ - .trimIndent() - ) + writePklFile(additionalExamples = examples) writeFile("test.pkl-expected.pcf", bigTestExpected) writeBuildFile("overwrite = true") @@ -173,6 +168,78 @@ class TestsTest : AbstractTest() { assertThat(output).contains("user 1 ✍️") } + @Test + fun `full example with error`() { + writeBuildFile() + + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") + + assertThat(output.trimStart()) + .startsWith( + """ + > Task :evalTest FAILED + module test + facts + Error: + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) + examples + ✅ user 0 + ❌ user 1 #0 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + ❌ user 1 #1 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + ❌ 25.0% tests pass [3/4 failed] + """ + .trimIndent() + ) + } + @Test fun `JUnit reports`() { val pklFile = writePklFile(contents = bigTest) @@ -189,15 +256,15 @@ class TestsTest : AbstractTest() { .isEqualTo( """ - - - - - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) + + + + + 4 == 9 (file:///file, line x) + "foo" == "bar" (file:///file, line x) - - + + (file:///file, line x) Expected: (file:///file, line x) new { @@ -210,7 +277,7 @@ class TestsTest : AbstractTest() { age = 41 } - + (file:///file, line x) Expected: (file:///file, line x) new { @@ -232,6 +299,102 @@ class TestsTest : AbstractTest() { ) } + @Test + fun `JUnit reports with error`() { + val pklFile = + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')") + + runTask("evalTest", expectFailure = true) + + val outputFile = testProjectDir.resolve("build/test.xml") + val report = outputFile.readText().stripFilesAndLines() + + assertThat(report) + .isEqualTo( + """ + + + + + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) + + + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + + + + """ + .trimIndent() + ) + } + + private val examples = + """ + ["user 0"] { + new { + name = "Cool" + age = 11 + } + } + ["user 1"] { + new { + name = "Pigeon" + age = 41 + } + new { + name = "Welma" + age = 35 + } + } + """ + .trimIndent() + private val bigTest = """ amends "pkl:test" @@ -254,22 +417,7 @@ class TestsTest : AbstractTest() { } examples { - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } + $examples } """ .trimIndent()