grader: Allow scorer to output percentage of subtask points (#641)

ia-toki · Aug 7, 2024 · 9c3dc60 · 9c3dc60
1 parent fa65e13
commit 9c3dc60
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 7 deletions.
diff --git a/...kends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java b/...kends/judgels-grader-engines/src/main/java/judgels/gabriel/aggregators/MinAggregator.java
@@ -23,7 +23,12 @@ public AggregationResult aggregate(List<TestCaseVerdict> testCaseVerdicts, doubl
 
             String points;
             if (verdict == Verdict.OK) {
-                double okPoints = testCaseVerdict.getPoints().orElse(0.0);
+                double okPoints = 0.0;
+                if (testCaseVerdict.getPercentage().isPresent()) {
+                    okPoints = testCaseVerdict.getPercentage().get() * subtaskPoints / 100.0;
+                } else if (testCaseVerdict.getPoints().isPresent()) {
+                    okPoints = testCaseVerdict.getPoints().get();
+                }
                 aggregatedPoints = Math.min(aggregatedPoints, okPoints);
                 points = "" + okPoints;
             } else if (verdict == Verdict.ACCEPTED) {

diff --git a/...ls-backends/judgels-grader-engines/src/main/java/judgels/gabriel/api/TestCaseVerdict.java b/...ls-backends/judgels-grader-engines/src/main/java/judgels/gabriel/api/TestCaseVerdict.java
@@ -7,6 +7,7 @@
 public interface TestCaseVerdict {
     Verdict getVerdict();
     Optional<Double> getPoints();
+    Optional<Integer> getPercentage();
     Optional<String> getFeedback();
 
     class Builder extends ImmutableTestCaseVerdict.Builder {}

diff --git a/...s/judgels-grader-engines/src/main/java/judgels/gabriel/helpers/TestCaseVerdictParser.java b/...s/judgels-grader-engines/src/main/java/judgels/gabriel/helpers/TestCaseVerdictParser.java
@@ -15,6 +15,7 @@ public TestCaseVerdict parseOutput(String output) throws ScoringException {
 
         Verdict verdict;
         Optional<Double> points = Optional.empty();
+        Optional<Integer> percentage = Optional.empty();
         Optional<String> feedback = Optional.empty();
 
         switch (lines[0]) {
@@ -39,11 +40,22 @@ public TestCaseVerdict parseOutput(String output) throws ScoringException {
                 if (tokens.length == 0) {
                     throw new ScoringException("Invalid <points> for OK: " + lines[1]);
                 }
-                try {
-                    points = Optional.of(Double.parseDouble(tokens[0]));
-                } catch (NumberFormatException e) {
-                    throw new ScoringException("Invalid <points> for OK: " + tokens[0]);
+                String result = tokens[0].trim();
+                if (!result.isEmpty() && result.charAt(result.length() - 1) == '%') {
+                    String percentageString = result.substring(0, result.length() - 1);
+                    try {
+                        percentage = Optional.of(Integer.parseInt(percentageString));
+                    } catch (NumberFormatException e) {
+                        throw new ScoringException("Invalid <percentage> for OK: " + result);
+                    }
+                } else {
+                    try {
+                        points = Optional.of(Double.parseDouble(result));
+                    } catch (NumberFormatException e) {
+                        throw new ScoringException("Invalid <points> for OK: " + result);
+                    }
                 }
+
                 if (tokens.length > 1) {
                     feedback = Optional.of(tokens[1]);
                 }
@@ -55,6 +67,7 @@ public TestCaseVerdict parseOutput(String output) throws ScoringException {
         return new TestCaseVerdict.Builder()
                 .verdict(verdict)
                 .points(points)
+                .percentage(percentage)
                 .feedback(feedback)
                 .build();
     }

diff --git a/.../judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java b/.../judgels-grader-engines/src/test/java/judgels/gabriel/aggregators/MinAggregatorTests.java
@@ -78,6 +78,18 @@ void aggregate_min_ok_points() {
         assertThat(result.getTestCasePoints()).containsExactly("*", "20.0", "30.0");
     }
 
+    @Test
+    void aggregate_min_ok_percentage() {
+        List<TestCaseVerdict> testCaseVerdicts = ImmutableList.of(
+                new TestCaseVerdict.Builder().verdict(Verdict.ACCEPTED).build(),
+                new TestCaseVerdict.Builder().verdict(Verdict.OK).percentage(25).build(),
+                new TestCaseVerdict.Builder().verdict(Verdict.OK).percentage(50).build());
+
+        AggregationResult result = aggregator.aggregate(testCaseVerdicts, 70.0);
+        assertThat(result.getSubtaskVerdict()).isEqualTo(SubtaskVerdict.of(Verdict.OK, 17.5));
+        assertThat(result.getTestCasePoints()).containsExactly("*", "17.5", "35.0");
+    }
+
     @Test
     void aggregate_empty() {
         List<TestCaseVerdict> testCaseVerdicts = ImmutableList.of();

diff --git a/...gels-grader-engines/src/test/java/judgels/gabriel/scorers/TestCaseVerdictParserTests.java b/...gels-grader-engines/src/test/java/judgels/gabriel/scorers/TestCaseVerdictParserTests.java
@@ -56,17 +56,29 @@ void wa() throws ScoringException {
         }
 
         @Test
-        void ok() throws ScoringException {
+        void ok_points() throws ScoringException {
             assertThat(parser.parseOutput("OK\n70\n")).isEqualTo(
                     new TestCaseVerdict.Builder().verdict(Verdict.OK).points(70.0).build());
         }
 
         @Test
-        void ok_with_feedback() throws ScoringException {
+        void ok_points_with_feedback() throws ScoringException {
             assertThat(parser.parseOutput("OK\n70 a feedback\n")).isEqualTo(
                     new TestCaseVerdict.Builder().verdict(Verdict.OK).points(70.0).feedback("a feedback").build());
         }
 
+        @Test
+        void ok_percentage() throws ScoringException {
+            assertThat(parser.parseOutput("OK\n25%\n")).isEqualTo(
+                    new TestCaseVerdict.Builder().verdict(Verdict.OK).percentage(25).build());
+        }
+
+        @Test
+        void ok_percentage_with_feedback() throws ScoringException {
+            assertThat(parser.parseOutput("OK\n25% a feedback\n")).isEqualTo(
+                    new TestCaseVerdict.Builder().verdict(Verdict.OK).percentage(25).feedback("a feedback").build());
+        }
+
         @Test
         void ok_failed_empty_second_line() {
             assertThatThrownBy(() -> parser.parseOutput("OK\n"))
@@ -80,6 +92,13 @@ void ok_failed_unknown_points_format() {
                     .isInstanceOf(ScoringException.class)
                     .hasMessage("Invalid <points> for OK: abc");
         }
+
+        @Test
+        void ok_failed_unknown_percentage_format() {
+            assertThatThrownBy(() -> parser.parseOutput("OK\nabc%\n"))
+                    .isInstanceOf(ScoringException.class)
+                    .hasMessage("Invalid <percentage> for OK: abc%");
+        }
     }
 
     @Nested