diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt index 79435e939..6b8eb8062 100644 --- a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt +++ b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt @@ -63,8 +63,11 @@ class CliTestRunnerTest { .isEqualTo( """ module test - succeed ✅ + facts + ✅ succeed + ✅ 100.0% tests pass [1 passed], 100.0% asserts pass [2 passed] + """ .trimIndent() ) @@ -80,7 +83,7 @@ class CliTestRunnerTest { facts { ["fail"] { 4 == 9 - "foo" == "bar" + "foo" != "bar" } } """ @@ -97,10 +100,12 @@ class CliTestRunnerTest { .isEqualTo( """ module test - fail ❌ - 4 == 9 ❌ - "foo" == "bar" ❌ + facts + ❌ fail + 4 == 9 + ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed] + """ .trimIndent() ) @@ -118,7 +123,8 @@ class CliTestRunnerTest { 9 == trace(9) "foo" == "foo" } - ["fail"] { + ["bar"] { + "foo" == "foo" 5 == 9 } } @@ -136,9 +142,9 @@ class CliTestRunnerTest { """ - - - 5 == 9 ❌ + + + 5 == 9 diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java index 9b4c149de..52fcb6c91 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestResults.java @@ -20,61 +20,40 @@ import java.util.Collections; import java.util.List; import org.pkl.core.PklException; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestSection; /** Aggregate test results for a module. Used to verify test failures and generate reports. */ public final class TestResults { - - private final String module; - private final String displayUri; - private final List results = new ArrayList<>(); + public final String moduleName; + public final String displayUri; + public final TestSectionResults module = new TestSectionResults(TestSection.MODULE); + public final TestSectionResults facts = new TestSectionResults(TestSection.FACTS); + public final TestSectionResults examples = new TestSectionResults(TestSection.EXAMPLES); private String err = ""; - public TestResults(String module, String displayUri) { - this.module = module; + public TestResults(String moduleName, String displayUri) { + this.moduleName = moduleName; this.displayUri = displayUri; } - public String getModuleName() { - return module; - } - - public String getDisplayUri() { - return displayUri; - } - - public List getResults() { - return Collections.unmodifiableList(results); - } - - public TestResult newResult(String name) { - var result = new TestResult(name); - results.add(result); - return result; + public int totalTests() { + return module.totalTests() + facts.totalTests() + examples.totalTests(); } - public void newResult(String name, Failure failure) { - var result = new TestResult(name); - result.addFailure(failure); - results.add(result); + public int totalFailures() { + return module.totalFailures() + facts.totalFailures() + examples.totalFailures(); } - public int totalTests() { - return results.size(); + public int totalAsserts() { + return module.totalAsserts() + facts.totalAsserts() + examples.totalAsserts(); } - public int totalFailures() { - int total = 0; - for (var res : results) { - total += res.getFailures().size(); - } - return total; + public int totalAssertsFailed() { + return module.totalAssertsFailed() + facts.totalAssertsFailed() + examples.totalAssertsFailed(); } public boolean failed() { - for (var res : results) { - if (res.isFailure()) return true; - } - return false; + return module.failed() || facts.failed() || examples.failed(); } public String getErr() { @@ -85,147 +64,277 @@ public void setErr(String err) { this.err = err; } - public static class TestResult { + public static class TestSectionResults { + public final TestSection name; + private final List results = new ArrayList<>(); + private Error error; - private final String name; - private final List failures = new ArrayList<>(); - private final List errors = new ArrayList<>(); - private boolean isExampleWritten = false; - - public TestResult(String name) { + public TestSectionResults(TestSection name) { this.name = name; } - public boolean isSuccess() { - return failures.isEmpty() && errors.isEmpty(); + public void setError(Error error) { + this.error = error; } - boolean isFailure() { - return !isSuccess(); + public Error getError() { + return error; } - public String getName() { - return name; + public boolean hasError() { + return error != null; } - public boolean isExampleWritten() { - return isExampleWritten; + public List getResults() { + return Collections.unmodifiableList(results); } - public void setExampleWritten(boolean exampleWritten) { - isExampleWritten = exampleWritten; + public TestResult newResult(String name) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + results.add(result); + return result; } - public List getFailures() { - return Collections.unmodifiableList(failures); + public void newResult(String name, Failure failure) { + var result = new TestResult(name, this.name == TestSection.EXAMPLES); + result.addFailure(failure); + results.add(result); } - public void addFailure(Failure description) { - failures.add(description); + public int totalTests() { + var total = results.size(); + return (hasError() ? ++total : total); } - public List getErrors() { - return Collections.unmodifiableList(errors); + public int totalAsserts() { + int total = 0; + for (var res : results) { + total += res.totalAsserts(); + } + return (hasError() ? ++total : total); } - public void addError(Error err) { - errors.add(err); + public int totalAssertsFailed() { + int total = 0; + for (var res : results) { + total += res.totalAssertsFailed(); + } + return (hasError() ? ++total : total); } - } - public static class Failure { - - private final String kind; - private final String rendered; - - private Failure(String kind, String rendered) { - this.kind = kind; - this.rendered = rendered; - } - - public String getKind() { - return kind; - } - - public String getRendered() { - return rendered; - } - - public static Failure buildFactFailure(SourceSection sourceSection, String description) { - return new Failure( - "Fact Failure", sourceSection.getCharacters() + " ❌ (" + description + ")"); - } - - public static Failure buildExampleLengthMismatchFailure( - String location, String property, int expectedLength, int actualLength) { - String builder = - "(" - + location - + ")\n" - + "Output mismatch: Expected \"" - + property - + "\" to contain " - + expectedLength - + " examples, but found " - + actualLength; - return new Failure("Output Mismatch (Length)", builder); - } - - public static Failure buildExamplePropertyMismatchFailure( - String location, String property, boolean isMissingInExpected) { - var builder = new StringBuilder(); - builder - .append("(") - .append(location) - .append(")\n") - .append("Output mismatch: \"") - .append(property); - if (isMissingInExpected) { - builder.append("\" exists in actual but not in expected output"); - } else { - builder.append("\" exists in expected but not in actual output"); - } - return new Failure("Output Mismatch", builder.toString()); - } - - public static Failure buildExampleFailure( - String location, - String expectedLocation, - String expectedValue, - String actualLocation, - String actualValue) { - String builder = - "(" - + location - + ")\n" - + "Expected: (" - + expectedLocation - + ")\n" - + expectedValue - + "\nActual: (" - + actualLocation - + ")\n" - + actualValue; - return new Failure("Example Failure", builder); + public int totalFailures() { + int total = 0; + for (var res : results) { + if (res.isFailure()) total++; + } + return (hasError() ? ++total : total); } - } - public static class Error { + public boolean failed() { + if (hasError()) return true; + + for (var res : results) { + if (res.isFailure()) return true; + } + return false; + } + + public static class TestResult { + public final String name; + private int totalAsserts = 0; + private int totalAssertsFailed = 0; + private final List failures = new ArrayList<>(); + public final boolean isExample; + private boolean isExampleWritten = false; + + public TestResult(String name, boolean isExample) { + this.name = name; + this.isExample = isExample; + } + + public boolean isSuccess() { + return failures.isEmpty(); + } + + public boolean isFailure() { + return !isSuccess(); + } + + public boolean isExampleWritten() { + return isExampleWritten; + } - private final String message; - private final PklException exception; + public void setExampleWritten(boolean exampleWritten) { + isExampleWritten = exampleWritten; + } + + public int totalAsserts() { + return totalAsserts; + } + + public void countAssert() { + totalAsserts++; + } - public Error(String message, PklException exception) { - this.message = message; - this.exception = exception; + public List getFailures() { + return Collections.unmodifiableList(failures); + } + + public int totalAssertsFailed() { + return totalAssertsFailed; + } + + public void addFailure(Failure description) { + failures.add(description); + totalAssertsFailed++; + } } - public String getMessage() { - return message; + public static class Failure { + + private final String kind; + private final String failure; + private final String location; + + private Failure(String kind, String failure, String location) { + this.kind = kind; + this.failure = failure; + this.location = location; + } + + public String getKind() { + return kind; + } + + public String getFailure() { + return failure; + } + + public String getLocation() { + return location; + } + + public static String renderLocation(String location) { + return "(" + location + ")"; + } + + public String getRendered() { + String rendered; + + if (kind == "Fact Failure") { + rendered = failure + " " + renderLocation(getLocation()); + } else { + rendered = renderLocation(getLocation()) + "\n" + failure; + } + + return rendered; + } + + public static Failure buildFactFailure(String location, SourceSection sourceSection) { + return new Failure("Fact Failure", sourceSection.getCharacters().toString(), location); + } + + public static Failure buildExampleLengthMismatchFailure( + String location, String property, int expectedLength, int actualLength) { + var builder = new StringBuilder(); + builder + .append("Output mismatch: Expected \"") + .append(property) + .append("\" to contain ") + .append(expectedLength) + .append(" examples, but found ") + .append(actualLength); + + return new Failure("Output Mismatch (Length)", builder.toString(), location); + } + + public static Failure buildExamplePropertyMismatchFailure( + String location, String property, boolean isMissingInExpected) { + + String exists_in; + String missing_in; + + if (isMissingInExpected) { + exists_in = "actual"; + missing_in = "expected"; + } else { + exists_in = "expected"; + missing_in = "actual"; + } + + var builder = new StringBuilder(); + builder + .append("Output mismatch: \"") + .append(property) + .append("\" exists in ") + .append(exists_in) + .append(" but not in ") + .append(missing_in) + .append(" output"); + + return new Failure("Output Mismatch", builder.toString(), location); + } + + public static Failure buildExampleFailure( + String location, + String expectedLocation, + String expectedValue, + String actualLocation, + String actualValue) { + var builder = new StringBuilder(); + builder + .append("Expected: ") + .append(renderLocation(expectedLocation)) + .append("\n") + .append(expectedValue) + .append("\n") + .append("Actual: ") + .append(renderLocation(actualLocation)) + .append("\n") + .append(actualValue); + + return new Failure("Example Failure", builder.toString(), location); + } } - public Exception getException() { - return exception; + public static class Error { + + private final String message; + private final PklException exception; + + public Error(String message, PklException exception) { + this.message = message; + this.exception = exception; + } + + public String getMessage() { + return message; + } + + public Exception getException() { + return exception; + } + + public String getRendered() { + return exception.getMessage(); + } + } + + public enum TestSection { + MODULE("module"), + FACTS("facts"), + EXAMPLES("examples"); + + private final String name; + + TestSection(final String name) { + this.name = name; + } + + @Override + public String toString() { + return name; + } } } } diff --git a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java index 47da5a83e..67f3658d2 100644 --- a/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java +++ b/pkl-core/src/main/java/org/pkl/core/runtime/TestRunner.java @@ -23,8 +23,9 @@ import org.pkl.core.StackFrameTransformer; import org.pkl.core.ast.member.ObjectMember; import org.pkl.core.module.ModuleKeys; -import org.pkl.core.runtime.TestResults.Error; -import org.pkl.core.runtime.TestResults.Failure; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.Failure; import org.pkl.core.stdlib.PklConverter; import org.pkl.core.stdlib.base.PcfRenderer; import org.pkl.core.util.EconomicMaps; @@ -51,12 +52,25 @@ public TestResults run(VmTyped testModule) { try { checkAmendsPklTest(testModule); - runFacts(testModule, results); - runExamples(testModule, info, results); } catch (VmException v) { - var meta = results.newResult(info.getModuleName()); - meta.addError(new Error(v.getMessage(), v.toPklException(stackFrameTransformer))); + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.module.setError(error); } + + try { + runFacts(testModule, results.facts); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.facts.setError(error); + } + + try { + runExamples(testModule, info, results.examples); + } catch (VmException v) { + var error = new Error(v.getMessage(), v.toPklException(stackFrameTransformer)); + results.examples.setError(error); + } + results.setErr(logger.getLogs()); return results; } @@ -72,7 +86,7 @@ private void checkAmendsPklTest(VmTyped value) { } } - private void runFacts(VmTyped testModule, TestResults results) { + private void runFacts(VmTyped testModule, TestSectionResults results) { var facts = VmUtils.readMember(testModule, Identifier.FACTS); if (facts instanceof VmNull) return; @@ -83,11 +97,13 @@ private void runFacts(VmTyped testModule, TestResults results) { var groupListing = (VmListing) groupValue; groupListing.forceAndIterateMemberValues( ((factIndex, factMember, factValue) -> { + result.countAssert(); + assert factValue instanceof Boolean; if (factValue == Boolean.FALSE) { result.addFailure( Failure.buildFactFailure( - factMember.getSourceSection(), getDisplayUri(factMember))); + getDisplayUri(factMember), factMember.getSourceSection())); } return true; })); @@ -95,7 +111,7 @@ private void runFacts(VmTyped testModule, TestResults results) { }); } - private void runExamples(VmTyped testModule, ModuleInfo info, TestResults results) { + private void runExamples(VmTyped testModule, ModuleInfo info, TestSectionResults results) { var examples = VmUtils.readMember(testModule, Identifier.EXAMPLES); if (examples instanceof VmNull) return; @@ -138,7 +154,10 @@ private void runExamples(VmTyped testModule, ModuleInfo info, TestResults result } private void doRunAndValidateExamples( - VmMapping examples, Path expectedOutputFile, Path actualOutputFile, TestResults results) { + VmMapping examples, + Path expectedOutputFile, + Path actualOutputFile, + TestSectionResults results) { var expectedExampleOutputs = loadExampleOutputs(expectedOutputFile); var actualExampleOutputs = new MutableReference(null); var allGroupsSucceeded = new MutableBoolean(true); @@ -202,8 +221,11 @@ private void doRunAndValidateExamples( .build(); } + var exampleName = + group.getLength() == 1 ? testName : testName + " #" + exampleIndex; + results.newResult( - testName, + exampleName, Failure.buildExampleFailure( getDisplayUri(exampleMember), getDisplayUri(expectedMember), @@ -244,10 +266,12 @@ private void doRunAndValidateExamples( } } - private void doRunAndWriteExamples(VmMapping examples, Path outputFile, TestResults results) { + private void doRunAndWriteExamples( + VmMapping examples, Path outputFile, TestSectionResults results) { examples.forceAndIterateMemberValues( (groupKey, groupMember, groupValue) -> { - results.newResult(String.valueOf(groupKey)).setExampleWritten(true); + var example = results.newResult(String.valueOf(groupKey)); + example.setExampleWritten(true); return true; }); writeExampleOutputs(outputFile, examples); diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java index c5c10eeff..ca8378c72 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/JUnitReport.java @@ -24,7 +24,9 @@ import org.pkl.core.runtime.BaseModule; import org.pkl.core.runtime.Identifier; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.Error; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.runtime.VmDynamic; import org.pkl.core.runtime.VmMapping; import org.pkl.core.runtime.VmTyped; @@ -41,27 +43,46 @@ public void report(TestResults results, Writer writer) throws IOException { writer.append(renderXML(" ", "1.0", buildSuite(results))); } - private VmDynamic buildSuite(TestResults res) { - var testCases = testCases(res); - if (!res.getErr().isBlank()) { + private VmDynamic buildSuite(TestResults results) { + var testCases = testCases(results.moduleName, results.facts); + testCases.addAll(testCases(results.moduleName, results.examples)); + + if (!results.getErr().isBlank()) { var err = buildXmlElement( "system-err", VmMapping.empty(), - members -> members.put("body", syntheticElement(makeCdata(res.getErr())))); + members -> members.put("body", syntheticElement(makeCdata(results.getErr())))); testCases.add(err); } - return buildXmlElement( - "testsuite", buildRootAttributes(res), testCases.toArray(new VmDynamic[0])); + + var attrs = + buildAttributes( + "name", results.moduleName, + "tests", (long) results.totalTests(), + "failures", (long) results.totalFailures()); + + return buildXmlElement("testsuite", attrs, testCases.toArray(new VmDynamic[0])); } - private ArrayList testCases(TestResults results) { - var className = results.getModuleName(); - var elements = new ArrayList(results.totalTests()); - for (var res : results.getResults()) { - var attrs = buildAttributes("classname", className, "name", res.getName()); + private ArrayList testCases(String moduleName, TestSectionResults testSectionResults) { + var elements = new ArrayList(testSectionResults.totalTests()); + + if (testSectionResults.hasError()) { + var error = error(testSectionResults.getError()); + + var attrs = + buildAttributes("classname", moduleName + "." + testSectionResults.name, "name", "error"); + var element = buildXmlElement("testcase", attrs, error.toArray(new VmDynamic[0])); + + elements.add(element); + } + + for (var res : testSectionResults.getResults()) { + var attrs = + buildAttributes( + "classname", moduleName + "." + testSectionResults.name, "name", res.name); var failures = failures(res); - failures.addAll(errors(res)); var element = buildXmlElement("testcase", attrs, failures.toArray(new VmDynamic[0])); elements.add(element); } @@ -83,19 +104,14 @@ private ArrayList failures(TestResult res) { return list; } - private ArrayList errors(TestResult res) { + private ArrayList error(Error error) { var list = new ArrayList(); - long i = 0; - for (var error : res.getErrors()) { - var attrs = buildAttributes("message", error.getMessage()); - long element = i++; - list.add( - buildXmlElement( - "error", - attrs, - members -> - members.put(element, syntheticElement(error.getException().getMessage())))); - } + var attrs = buildAttributes("message", error.getMessage()); + list.add( + buildXmlElement( + "error", + attrs, + members -> members.put(1, syntheticElement("\n" + error.getRendered())))); return list; } @@ -130,16 +146,6 @@ private VmDynamic buildXmlElement( members.size() - 4); } - private VmMapping buildRootAttributes(TestResults results) { - return buildAttributes( - "name", - results.getModuleName(), - "tests", - (long) results.totalTests(), - "failures", - (long) results.totalFailures()); - } - private VmMapping buildAttributes(Object... attributes) { EconomicMap attrs = EconomicMaps.create(attributes.length); for (int i = 0; i < attributes.length; i += 2) { diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java index 82e788cef..ce9db655c 100644 --- a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java +++ b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java @@ -19,8 +19,8 @@ import java.io.Writer; import java.util.stream.Collectors; import org.pkl.core.runtime.TestResults; -import org.pkl.core.runtime.TestResults.Failure; -import org.pkl.core.runtime.TestResults.TestResult; +import org.pkl.core.runtime.TestResults.TestSectionResults; +import org.pkl.core.runtime.TestResults.TestSectionResults.TestResult; import org.pkl.core.util.StringUtils; public final class SimpleReport implements TestReport { @@ -28,38 +28,66 @@ public final class SimpleReport implements TestReport { @Override public void report(TestResults results, Writer writer) throws IOException { var builder = new StringBuilder(); - builder.append("module "); - builder.append(results.getModuleName()); - builder.append(" (").append(results.getDisplayUri()).append(")\n"); - StringUtils.joinToStringBuilder( - builder, results.getResults(), "\n", res -> reportResult(res, builder)); - builder.append("\n"); + + builder.append("module ").append(results.moduleName).append("\n"); + + reportResults(results.facts, builder); + reportResults(results.examples, builder); + + builder.append(results.failed() ? "❌ " : "✅ "); + + var totalStatsLine = + makeStatsLine("tests", results.totalTests(), results.totalFailures(), results.failed()); + builder.append(totalStatsLine); + + var totalAssertsStatsLine = + makeStatsLine( + "asserts", results.totalAsserts(), results.totalAssertsFailed(), results.failed()); + builder.append(", ").append(totalAssertsStatsLine); + + builder.append("\n\n"); + writer.append(builder); } - private void reportResult(TestResult result, StringBuilder builder) { - builder.append(" ").append(result.getName()); - if (result.isExampleWritten()) { - builder.append(" ✍️"); - } else if (result.isSuccess()) { - builder.append(" ✅"); - } else { - builder.append(" ❌\n"); - StringUtils.joinToStringBuilder( - builder, result.getFailures(), "\n", failure -> reportFailure(failure, builder)); + private void reportResults(TestSectionResults section, StringBuilder builder) { + if (!section.getResults().isEmpty()) { + builder.append(" ").append(section.name).append("\n"); + StringUtils.joinToStringBuilder( - builder, - result.getErrors(), - "\n", - error -> { - builder.append(" Error:\n"); - appendPadded(builder, error.getException().getMessage(), " "); - }); + builder, section.getResults(), "\n", res -> reportResult(res, builder)); + builder.append("\n"); + } else if (section.hasError()) { + builder.append(" ").append(section.name).append("\n"); + var error = "Error:\n" + section.getError().getRendered(); + appendPadded(builder, error, " "); + builder.append("\n"); } } - public static void reportFailure(Failure failure, StringBuilder builder) { - appendPadded(builder, failure.getRendered(), " "); + private void reportResult(TestResult result, StringBuilder builder) { + builder.append(" "); + + if (result.isExampleWritten()) { + builder.append(result.name).append(" ✍️"); + } else { + builder.append(result.isFailure() ? "❌ " : "✅ ").append(result.name); + + if (!result.isExample) { + var statsLine = + makeStatsLine( + "asserts", result.totalAsserts(), result.getFailures().size(), result.isFailure()); + } + + if (result.isFailure()) { + builder.append("\n"); + StringUtils.joinToStringBuilder( + builder, + result.getFailures(), + "\n", + failure -> appendPadded(builder, failure.getRendered(), " ")); + } + } } private static void appendPadded(StringBuilder builder, String lines, String padding) { @@ -69,4 +97,19 @@ private static void appendPadded(StringBuilder builder, String lines, String pad "\n", str -> builder.append(padding).append(str)); } + + private String makeStatsLine(String kind, int total, int failed, boolean isFailed) { + var passed = total - failed; + var pct_passed = total > 0 ? 100.0 * passed / total : 0.0; + + String line = String.format("%.1f%% %s pass", pct_passed, kind); + + if (isFailed) { + line += String.format(" [%d/%d failed]", failed, total); + } else { + line += String.format(" [%d passed]", passed); + } + + return line; + } } diff --git a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt index 15f99d6b6..bc0c8c5fd 100644 --- a/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt +++ b/pkl-core/src/test/kotlin/org/pkl/core/EvaluateTestsTest.kt @@ -54,7 +54,7 @@ class EvaluateTestsTest { assertThat(results.displayUri).isEqualTo("repl:text") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("should pass") + assertThat(results.facts.results[0].name).isEqualTo("should pass") assertThat(results.err.isBlank()).isTrue } @@ -79,18 +79,19 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] + val res = results.facts.results[0] assertThat(res.name).isEqualTo("should fail") - assertThat(res.errors).isEmpty() + assertThat(results.facts.hasError()).isFalse + assertThat(res.failures.size).isEqualTo(2) val fail1 = res.failures[0] - assertThat(fail1.rendered).isEqualTo("1 == 2 ❌ (repl:text)") + assertThat(fail1.rendered).isEqualTo("1 == 2 (repl:text)") val fail2 = res.failures[1] - assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" ❌ (repl:text)""") + assertThat(fail2.rendered).isEqualTo(""""foo" == "bar" (repl:text)""") } @Test @@ -114,15 +115,14 @@ class EvaluateTestsTest { ) assertThat(results.totalTests()).isEqualTo(1) - assertThat(results.totalFailures()).isEqualTo(0) + assertThat(results.totalFailures()).isEqualTo(1) assertThat(results.failed()).isTrue - val res = results.results[0] - assertThat(res.name).isEqualTo("text") - assertThat(res.failures).isEmpty() - assertThat(res.errors.size).isEqualTo(1) + val res = results.facts + assertThat(res.results).isEmpty() + assertThat(res.hasError()).isTrue - val error = res.errors[0] + val error = res.error assertThat(error.message).isEqualTo("got an error") assertThat(error.exception.message) .isEqualTo( @@ -183,7 +183,114 @@ class EvaluateTestsTest { assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") assertThat(results.totalTests()).isEqualTo(1) assertThat(results.failed()).isFalse - assertThat(results.results[0].name).isEqualTo("user") + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact failures with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + 1 == 2 + "foo" == "bar" + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results[0].name).isEqualTo("should fail") + assertThat(results.facts.results[0].failures.size).isEqualTo(2) + assertThat(results.examples.results[0].name).isEqualTo("user") + } + + @Test + fun `test fact error with successful example`(@TempDir tempDir: Path) { + val file = tempDir.createTempFile(prefix = "example", suffix = ".pkl") + Files.writeString( + file, + """ + amends "pkl:test" + + facts { + ["should fail"] { + throw("exception") + } + } + + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + Files.writeString( + createExpected(file), + """ + examples { + ["user"] { + new { + name = "Bob" + age = 33 + } + } + } + """ + .trimIndent() + ) + + val results = evaluator.evaluateTest(path(file), false) + assertThat(results.moduleName).startsWith("example") + assertThat(results.displayUri).startsWith("file:///").endsWith(".pkl") + assertThat(results.totalTests()).isEqualTo(2) + assertThat(results.totalFailures()).isEqualTo(1) + assertThat(results.failed()).isTrue + + assertThat(results.facts.results).isEmpty() + assertThat(results.facts.hasError()).isTrue + assertThat(results.examples.results[0].name).isEqualTo("user") } @Test @@ -228,9 +335,9 @@ class EvaluateTestsTest { assertThat(results.failed()).isTrue assertThat(results.totalFailures()).isEqualTo(1) - val res = results.results[0] + val res = results.examples.results[0] assertThat(res.name).isEqualTo("user") - assertThat(res.errors.isEmpty()).isTrue + assertFalse(results.examples.hasError()) val fail1 = res.failures[0] assertThat(fail1.rendered.stripFileAndLines(tempDir)) diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt index b5a77a954..009a6ee00 100644 --- a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt +++ b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt @@ -30,7 +30,7 @@ class TestsTest : AbstractTest() { writePklFile() val res = runTask("evalTest") - assertThat(res.output).contains("should pass ✅") + assertThat(res.output).contains("✅ should pass") } @Test @@ -49,9 +49,9 @@ class TestsTest : AbstractTest() { ) val res = runTask("evalTest", expectFailure = true) - assertThat(res.output).contains("should fail ❌") - assertThat(res.output).contains("1 == 3 ❌") - assertThat(res.output).contains(""""foo" == "bar" ❌""") + assertThat(res.output).contains("❌ should fail") + assertThat(res.output).contains("1 == 3") + assertThat(res.output).contains(""""foo" == "bar"""") } @Test @@ -68,24 +68,30 @@ class TestsTest : AbstractTest() { .trimIndent() ) - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") - assertThat(output) - .contains( + assertThat(output.trimStart()) + .startsWith( """ - module test (file:///file, line x) - test ❌ + > Task :evalTest FAILED + module test + facts Error: - –– Pkl Error –– - exception - - 9 | throw("exception") - ^^^^^^^^^^^^^^^^^^ - at test#facts["error"][#1] (file:///file, line x) - - 3 | facts { - ^^^^^^^ - at test#facts (file:///file, line x) + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) """ .trimIndent() ) @@ -98,42 +104,52 @@ class TestsTest : AbstractTest() { writeBuildFile() - val output = runTask("evalTest", expectFailure = true).output.stripFilesAndLines() + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") assertThat(output.trimStart()) - .contains( + .startsWith( """ - module test (file:///file, line x) - sum numbers ✅ - divide numbers ✅ - fail ❌ - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) - user 0 ✅ - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Pigeon" - age = 40 - } - Actual: (file:///file, line x) - new { - name = "Pigeon" - age = 41 - } - user 1 ❌ - (file:///file, line x) - Expected: (file:///file, line x) - new { - name = "Parrot" - age = 35 - } - Actual: (file:///file, line x) - new { - name = "Welma" - age = 35 - } + > Task :evalTest FAILED + pkl: TRACE: 8 = 8 (file:///file, line x) + module test + facts + ✅ sum numbers + ✅ divide numbers + ❌ fail + 4 == 9 (file:///file, line x) + "foo" == "bar" (file:///file, line x) + examples + ✅ user 0 + ❌ user 1 #0 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + ❌ user 1 #1 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + ❌ 50.0% tests pass [3/6 failed] """ .trimIndent() ) @@ -141,28 +157,7 @@ class TestsTest : AbstractTest() { @Test fun `overwrite expected examples`() { - writePklFile( - additionalExamples = - """ - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } - """ - .trimIndent() - ) + writePklFile(additionalExamples = examples) writeFile("test.pkl-expected.pcf", bigTestExpected) writeBuildFile("overwrite = true") @@ -173,6 +168,78 @@ class TestsTest : AbstractTest() { assertThat(output).contains("user 1 ✍️") } + @Test + fun `full example with error`() { + writeBuildFile() + + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + val output = + runTask("evalTest", expectFailure = true) + .output + .stripFilesAndLines() + .lineSequence() + .joinToString("\n") + + assertThat(output.trimStart()) + .startsWith( + """ + > Task :evalTest FAILED + module test + facts + Error: + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) + examples + ✅ user 0 + ❌ user 1 #0 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + ❌ user 1 #1 + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + ❌ 25.0% tests pass [3/4 failed] + """ + .trimIndent() + ) + } + @Test fun `JUnit reports`() { val pklFile = writePklFile(contents = bigTest) @@ -189,15 +256,15 @@ class TestsTest : AbstractTest() { .isEqualTo( """ - - - - - 4 == 9 ❌ (file:///file, line x) - "foo" == "bar" ❌ (file:///file, line x) + + + + + 4 == 9 (file:///file, line x) + "foo" == "bar" (file:///file, line x) - - + + (file:///file, line x) Expected: (file:///file, line x) new { @@ -210,7 +277,7 @@ class TestsTest : AbstractTest() { age = 41 } - + (file:///file, line x) Expected: (file:///file, line x) new { @@ -232,6 +299,102 @@ class TestsTest : AbstractTest() { ) } + @Test + fun `JUnit reports with error`() { + val pklFile = + writePklFile( + additionalFacts = + """ + ["error"] { + throw("exception") + } + """ + .trimIndent(), + additionalExamples = examples + ) + writeFile("test.pkl-expected.pcf", bigTestExpected) + + writeBuildFile("junitReportsDir = file('${pklFile.parent.toNormalizedPathString()}/build')") + + runTask("evalTest", expectFailure = true) + + val outputFile = testProjectDir.resolve("build/test.xml") + val report = outputFile.readText().stripFilesAndLines() + + assertThat(report) + .isEqualTo( + """ + + + + + –– Pkl Error –– + exception + + 9 | throw("exception") + ^^^^^^^^^^^^^^^^^^ + at test#facts["error"][#1] (file:///file, line x) + + 3 | facts { + ^^^^^^^ + at test#facts (file:///file, line x) + + + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Pigeon" + age = 40 + } + Actual: (file:///file, line x) + new { + name = "Pigeon" + age = 41 + } + + + (file:///file, line x) + Expected: (file:///file, line x) + new { + name = "Parrot" + age = 35 + } + Actual: (file:///file, line x) + new { + name = "Welma" + age = 35 + } + + + + """ + .trimIndent() + ) + } + + private val examples = + """ + ["user 0"] { + new { + name = "Cool" + age = 11 + } + } + ["user 1"] { + new { + name = "Pigeon" + age = 41 + } + new { + name = "Welma" + age = 35 + } + } + """ + .trimIndent() + private val bigTest = """ amends "pkl:test" @@ -254,22 +417,7 @@ class TestsTest : AbstractTest() { } examples { - ["user 0"] { - new { - name = "Cool" - age = 11 - } - } - ["user 1"] { - new { - name = "Pigeon" - age = 41 - } - new { - name = "Welma" - age = 35 - } - } + $examples } """ .trimIndent()