grader: show SKIPPED for subtasks without submitted files in OO probl…

…ems (#658)
ia-toki · Aug 26, 2024 · c4f851b · c4f851b
1 parent 0b29a7a
commit c4f851b
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 7 deletions.
diff --git a/...dgels/gabriel/engines/outputonly/OutputOnlyWithSubtasksGradingEngineIntegrationTests.java b/...dgels/gabriel/engines/outputonly/OutputOnlyWithSubtasksGradingEngineIntegrationTests.java
@@ -142,7 +142,7 @@ void wa_30_because_some_files_missing() throws GradingException {
                                 testCaseResult(SKIPPED, "?", Optional.empty(), 2))),
                 ImmutableList.of(
                         subtaskResult(1, ACCEPTED, 30),
-                        subtaskResult(2, OK, 0)));
+                        subtaskResult(2, SKIPPED, 0)));
     }
 
     @Test

diff --git a/...kends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java b/...kends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java
@@ -44,9 +44,6 @@ public AggregationResult aggregate(List<TestCaseVerdict> testCaseVerdicts, doubl
 
             testCasePoints.add(points);
         }
-        if (aggregatedVerdict == Verdict.SKIPPED) {
-            aggregatedVerdict = Verdict.OK;
-        }
 
         return new AggregationResult.Builder()
                 .subtaskVerdict(SubtaskVerdict.of(aggregatedVerdict, aggregatedPoints))

diff --git a/...s/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/SubtaskAggregator.java b/...s/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/SubtaskAggregator.java
@@ -17,6 +17,13 @@ public SubtaskVerdict aggregate(List<SubtaskVerdict> subtaskVerdicts) {
 
             aggregatedPoints += subtaskVerdict.getPoints();
         }
+
+        // This case can only logically happen for subtasks in an output-only problem,
+        // where the SKIPPED verdicts are due to unsubmitted output files.
+        if (aggregatedVerdict == Verdict.SKIPPED) {
+            aggregatedVerdict = Verdict.OK;
+        }
+
         return SubtaskVerdict.of(aggregatedVerdict, aggregatedPoints);
     }
 }
diff --git a/.../judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java b/.../judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java
@@ -62,7 +62,7 @@ void aggregate_zero_points_with_accepted_and_skipped() {
                 new TestCaseVerdict.Builder().verdict(Verdict.SKIPPED).build());
 
         AggregationResult result = aggregator.aggregate(testCaseVerdicts, 70.0);
-        assertThat(result.getSubtaskVerdict()).isEqualTo(SubtaskVerdict.of(Verdict.OK, 0.0));
+        assertThat(result.getSubtaskVerdict()).isEqualTo(SubtaskVerdict.of(Verdict.SKIPPED, 0.0));
         assertThat(result.getTestCasePoints()).containsExactly("*", "*", "?");
     }
 

diff --git a/...gels-grader-engines/src/test/java/judgels/gabriel/aggregators/SubtaskAggregatorTests.java b/...gels-grader-engines/src/test/java/judgels/gabriel/aggregators/SubtaskAggregatorTests.java
@@ -2,7 +2,6 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
-import com.google.common.collect.ImmutableList;
 import java.util.List;
 import judgels.gabriel.api.SubtaskVerdict;
 import judgels.gabriel.api.Verdict;
@@ -19,12 +18,22 @@ void before() {
 
     @Test
     void aggregate() {
-        List<SubtaskVerdict> subtaskVerdicts = ImmutableList.of(
+        List<SubtaskVerdict> subtaskVerdicts = List.of(
                 new SubtaskVerdict.Builder().verdict(Verdict.ACCEPTED).points(10).build(),
                 new SubtaskVerdict.Builder().verdict(Verdict.TIME_LIMIT_EXCEEDED).points(20).build(),
                 new SubtaskVerdict.Builder().verdict(Verdict.OK).points(30).build(),
+                new SubtaskVerdict.Builder().verdict(Verdict.SKIPPED).points(0).build(),
                 new SubtaskVerdict.Builder().verdict(Verdict.WRONG_ANSWER).points(0).build());
 
         assertThat(aggregator.aggregate(subtaskVerdicts)).isEqualTo(SubtaskVerdict.of(Verdict.TIME_LIMIT_EXCEEDED, 60));
     }
+
+    @Test
+    void aggregate_skipped() {
+        List<SubtaskVerdict> subtaskVerdicts = List.of(
+                new SubtaskVerdict.Builder().verdict(Verdict.SKIPPED).points(0).build(),
+                new SubtaskVerdict.Builder().verdict(Verdict.SKIPPED).points(0).build());
+
+        assertThat(aggregator.aggregate(subtaskVerdicts)).isEqualTo(SubtaskVerdict.of(Verdict.OK, 0));
+    }
 }