From c4f851bc8d0b89667c46cd9da73180834e8e1277 Mon Sep 17 00:00:00 2001 From: Ashar Fuadi Date: Mon, 26 Aug 2024 12:24:19 +0700 Subject: [PATCH] grader: show SKIPPED for subtasks without submitted files in OO problems (#658) --- ...lyWithSubtasksGradingEngineIntegrationTests.java | 2 +- .../judgels/gabriel/aggregators/MinAggregator.java | 3 --- .../gabriel/aggregators/SubtaskAggregator.java | 7 +++++++ .../gabriel/aggregators/MinAggregatorTests.java | 2 +- .../gabriel/aggregators/SubtaskAggregatorTests.java | 13 +++++++++++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/judgels-backends/judgels-grader-engines/src/integTest/java/judgels/gabriel/engines/outputonly/OutputOnlyWithSubtasksGradingEngineIntegrationTests.java b/judgels-backends/judgels-grader-engines/src/integTest/java/judgels/gabriel/engines/outputonly/OutputOnlyWithSubtasksGradingEngineIntegrationTests.java index 1b1b89ea8..a315a5883 100644 --- a/judgels-backends/judgels-grader-engines/src/integTest/java/judgels/gabriel/engines/outputonly/OutputOnlyWithSubtasksGradingEngineIntegrationTests.java +++ b/judgels-backends/judgels-grader-engines/src/integTest/java/judgels/gabriel/engines/outputonly/OutputOnlyWithSubtasksGradingEngineIntegrationTests.java @@ -142,7 +142,7 @@ void wa_30_because_some_files_missing() throws GradingException { testCaseResult(SKIPPED, "?", Optional.empty(), 2))), ImmutableList.of( subtaskResult(1, ACCEPTED, 30), - subtaskResult(2, OK, 0))); + subtaskResult(2, SKIPPED, 0))); } @Test diff --git a/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java b/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java index 8a70d6b8c..6ef50b3e3 100644 --- a/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java +++ b/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java @@ -44,9 +44,6 @@ public AggregationResult aggregate(List testCaseVerdicts, doubl testCasePoints.add(points); } - if (aggregatedVerdict == Verdict.SKIPPED) { - aggregatedVerdict = Verdict.OK; - } return new AggregationResult.Builder() .subtaskVerdict(SubtaskVerdict.of(aggregatedVerdict, aggregatedPoints)) diff --git a/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/SubtaskAggregator.java b/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/SubtaskAggregator.java index 5499073d5..7e544ca79 100644 --- a/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/SubtaskAggregator.java +++ b/judgels-backends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/SubtaskAggregator.java @@ -17,6 +17,13 @@ public SubtaskVerdict aggregate(List subtaskVerdicts) { aggregatedPoints += subtaskVerdict.getPoints(); } + + // This case can only logically happen for subtasks in an output-only problem, + // where the SKIPPED verdicts are due to unsubmitted output files. + if (aggregatedVerdict == Verdict.SKIPPED) { + aggregatedVerdict = Verdict.OK; + } + return SubtaskVerdict.of(aggregatedVerdict, aggregatedPoints); } } diff --git a/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java b/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java index 6b63f47b0..71159bedf 100644 --- a/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java +++ b/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java @@ -62,7 +62,7 @@ void aggregate_zero_points_with_accepted_and_skipped() { new TestCaseVerdict.Builder().verdict(Verdict.SKIPPED).build()); AggregationResult result = aggregator.aggregate(testCaseVerdicts, 70.0); - assertThat(result.getSubtaskVerdict()).isEqualTo(SubtaskVerdict.of(Verdict.OK, 0.0)); + assertThat(result.getSubtaskVerdict()).isEqualTo(SubtaskVerdict.of(Verdict.SKIPPED, 0.0)); assertThat(result.getTestCasePoints()).containsExactly("*", "*", "?"); } diff --git a/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/SubtaskAggregatorTests.java b/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/SubtaskAggregatorTests.java index 9adc29da2..8a6da20dd 100644 --- a/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/SubtaskAggregatorTests.java +++ b/judgels-backends/judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/SubtaskAggregatorTests.java @@ -2,7 +2,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.google.common.collect.ImmutableList; import java.util.List; import judgels.gabriel.api.SubtaskVerdict; import judgels.gabriel.api.Verdict; @@ -19,12 +18,22 @@ void before() { @Test void aggregate() { - List subtaskVerdicts = ImmutableList.of( + List subtaskVerdicts = List.of( new SubtaskVerdict.Builder().verdict(Verdict.ACCEPTED).points(10).build(), new SubtaskVerdict.Builder().verdict(Verdict.TIME_LIMIT_EXCEEDED).points(20).build(), new SubtaskVerdict.Builder().verdict(Verdict.OK).points(30).build(), + new SubtaskVerdict.Builder().verdict(Verdict.SKIPPED).points(0).build(), new SubtaskVerdict.Builder().verdict(Verdict.WRONG_ANSWER).points(0).build()); assertThat(aggregator.aggregate(subtaskVerdicts)).isEqualTo(SubtaskVerdict.of(Verdict.TIME_LIMIT_EXCEEDED, 60)); } + + @Test + void aggregate_skipped() { + List subtaskVerdicts = List.of( + new SubtaskVerdict.Builder().verdict(Verdict.SKIPPED).points(0).build(), + new SubtaskVerdict.Builder().verdict(Verdict.SKIPPED).points(0).build()); + + assertThat(aggregator.aggregate(subtaskVerdicts)).isEqualTo(SubtaskVerdict.of(Verdict.OK, 0)); + } }