Merge pull request #493 from dodona-edu/enhance/problem-tweaks

Tweak exception handling in problem statements

tested/descriptions/renderer.py

@@ -1,7 +1,6 @@
 """
 A Marko renderer that only renders TESTed code; all other things are left alone.
 """
-from doctest import DocTestParser
 
 from marko import block
 from marko.md_renderer import MarkdownRenderer
@@ -10,9 +9,7 @@
 from tested.dsl import parse_dsl, parse_string
 from tested.judge.evaluation import Channel, guess_expected_value, should_show
 from tested.languages.generation import generate_statement, get_readable_input
-from tested.testsuite import OutputChannel, Testcase
-
-TESTED_EXAMPLE_FORMAT = "console?lang=tested"
+from tested.testsuite import ExceptionOutputChannel, OutputChannel, Testcase
 
 
 def render_one_statement(bundle: Bundle, statement: str) -> str:
@@ -31,6 +28,17 @@ def _add_output(
 ):
     if should_show(output, channel):
         expected = guess_expected_value(bundle, output)
+        # Special handling of exceptions
+        if channel == Channel.EXCEPTION:
+            assert isinstance(output, ExceptionOutputChannel)
+            if output.exception and not output.exception.get_type(
+                bundle.config.programming_language
+            ):
+                name = bundle.language.get_declaration_metadata().get(
+                    "exception", "Exception"
+                )
+                expected = f"{name}: {expected}"
+
         results.append(expected)
 
 
@@ -48,35 +56,6 @@ def get_expected_output(bundle: Bundle, tc: Testcase) -> list[str]:
 
 class TestedRenderer(MarkdownRenderer):
     bundle: Bundle
-    _doctest_parser: DocTestParser
-
-    def __init__(self):
-        super().__init__()
-        self._doctest_parser = DocTestParser()
-
-    def _render_doctest(self, element: block.FencedCode) -> str:
-        """
-        Render a "doctest" code block.
-        """
-        assert element.lang == TESTED_EXAMPLE_FORMAT
-        raw_code = self.render_children(element)
-        doctests = self._doctest_parser.get_examples(raw_code)
-
-        resulting_lines = []
-        prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">")
-
-        # Both the doctests and the results are parsed as values in the DSL.
-        for examples in doctests:
-            generated_statement = render_one_statement(self.bundle, examples.source)
-            resulting_lines.append(f"{prompt} {generated_statement}")
-            resulting_lines.append(examples.want.lstrip())
-
-        language = (
-            f"console?lang={self.bundle.config.programming_language}&prompt={prompt}"
-        )
-        body = "\n".join(resulting_lines)
-
-        return f"```{language}\n{body}```\n"
 
     def _render_normal_statements(self, element: block.FencedCode) -> str:
         """
@@ -133,7 +112,5 @@ def render_fenced_code(self, element: block.FencedCode) -> str:
             return self._render_normal_statements(element)
         elif element.lang == "dsl":
             return self._render_dsl_statements(element)
-        elif element.lang == TESTED_EXAMPLE_FORMAT:
-            return self._render_doctest(element)
         else:
             return super().render_fenced_code(element)

tested/languages/config.py

@@ -41,6 +41,7 @@ class TypeDeclarationMetadata(TypedDict):
     nested_overrides: NotRequired[dict[AllTypes, tuple[str, str]]]
     prompt: NotRequired[str]
     natural_overrides: NotRequired[dict[str, dict[AllTypes, tuple[str, str]]]]
+    exception: NotRequired[str]
 
 
 class Language(ABC):

tested/languages/csharp/config.py

@@ -266,7 +266,8 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "list": "List",
                 "tuple": "Tuple",
                 "any": "Object",
-            }
+            },
+            "exception": "Exception",
         }
 
     def is_void_method(self, name: str) -> bool:

tested/languages/haskell/config.py

@@ -209,4 +209,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "sequence": True,
             },
             "nested_overrides": {"tuple": ("(", ")")},  # type: ignore
+            "exception": "Exception",
         }

tested/languages/java/config.py

@@ -207,6 +207,7 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
             },
             "nested": ("<", ">"),
             "nested_overrides": {"array": ("[", "]")},  # type: ignore
+            "exception": "Exception",
         }
 
     def is_void_method(self, name: str) -> bool:

tested/languages/javascript/config.py

@@ -235,4 +235,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "any": "object",
             },
             "nested": ("<", ">"),
+            "exception": "Error",
         }

tested/languages/kotlin/config.py

@@ -262,4 +262,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "any": "Any",
             },
             "nested": ("<", ">"),
+            "exception": "Exception",
         }

tested/languages/python/config.py

@@ -262,4 +262,5 @@ def get_declaration_metadata(self) -> TypeDeclarationMetadata:
                 "any": "Any",
             },
             "prompt": ">>>",
+            "exception": "Exception",
         }

tests/descriptions/example.haskell.md

@@ -6,7 +6,3 @@ aFunctionCall "yes"
 ```
 
 This is haskell.
-
-```console?lang=haskell&prompt=>
-> aFunctionCall "yes"
-```

tests/descriptions/example.java.md

@@ -6,7 +6,3 @@ Submission.aFunctionCall("yes")
 ```
 
 This is java.
-
-```console?lang=java&prompt=>
-> Submission.aFunctionCall("yes")
-```

tests/descriptions/example.kotlin.md

@@ -6,7 +6,3 @@ aFunctionCall("yes")
 ```
 
 This is kotlin.
-
-```console?lang=kotlin&prompt=>
-> aFunctionCall("yes")
-```

tests/descriptions/example.md.jinja2

@@ -15,6 +15,3 @@ This is java.
 This is haskell.
 {% endif %}
 
-```console?lang=tested
->>> a_function_call("yes")
-```

tests/descriptions/example.python.md

@@ -6,7 +6,3 @@ a_function_call('yes')
 ```
 
 This is python.
-
-```console?lang=python&prompt=>>>
->>> a_function_call('yes')
-```

tests/descriptions/recoupling.md.j2

@@ -1,14 +1,14 @@
 Write a function `{{ function('divide') }}` that takes two arguments:
 _i_) a word (`{{ datatype("text")}}`) and
 _ii_) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`{{datatype("integer")}}`) into which the word must be divided.
-If the word passed to the function `{{function('divide')}}` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If the word passed to the function `{{function('divide')}}` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a {{datatype("list").singular}} (`{{datatype("list")}}`) containing the $$n$$ groups (`{{datatype("text")}}`) into which the given word can be divided.
 All groups need to have the same length (same number of letters).
 
 Write another function `{{function('recouple')}}` that takes two arguments:
 _i_) a {{datatype("sequence").singular}} (`{{datatype("sequence")}}`) of $$m \\in \\mathbb{N}\_0$$ words (`{{datatype("text")}}`) and
 _ii_) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`{{datatype("integer")}}`) into which the words must be divided.
-If at least one of the words passed to the function `{{function('recouple')}}` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If at least one of the words passed to the function `{{function('recouple')}}` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a {{datatype("sequence").singular}} containing the $$n$$ new words (`{{datatype("text")}}`) obtained when each of the $$m$$ given words is divided into $$n$$ groups that have the same length, and if each of the $$m$$ corresponding groups is merged into a new word.
 The type of the returned {{datatype("sequence").singular}} (`{{datatype("sequence")}}`) must correspond to the type of the {{datatype("sequence").singular}} passed as a first argument to the function.
 
@@ -23,23 +23,13 @@ units:
     - expression: "divide('COMMUNED', 4)"
       return: ["CO", "MM", "UN", "ED"]
     - expression: "divide('programming', 5)"
-      exception:
-        message: "invalid division"
-        types:
-          python: AssertionError
-          javascript: AssertionError
-          java: IllegalArgumentException
+      exception: "invalid division"
 - unit: "Recouple"
   scripts:
     - expression: "recouple(['ACcoST', 'COmmIT', 'LAunCH', 'DEedED'], 3)"
       return: !list ["ACCOLADE", "communed", "STITCHED"]
     - expression: "recouple(('ACCOLADE', 'communed', 'STITCHED'), 4)"
       return: !tuple ["ACcoST", "COmmIT", "LAunCH", "DEedED"]
     - expression: "recouple(['programming', 'computer', 'games'], 5)"
-      exception:
-        message: "invalid division"
-        types:
-          python: AssertionError
-          javascript: AssertionError
-          java: IllegalArgumentException
+      exception: "invalid division"
 ```

tests/descriptions/recoupling.python.md

@@ -1,14 +1,14 @@
 Write a function `divide` that takes two arguments:
 *i*) a word (`str`) and
 *ii*) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`int`) into which the word must be divided.
-If the word passed to the function `divide` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If the word passed to the function `divide` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a list (`list`) containing the $$n$$ groups (`str`) into which the given word can be divided.
 All groups need to have the same length (same number of letters).
 
 Write another function `recouple` that takes two arguments:
 *i*) a sequence (`list` or `tuple`) of $$m \\in \\mathbb{N}\_0$$ words (`str`) and
 *ii*) the number of (non-overlapping) groups $$n \\in \\mathbb{N}\_0$$ (`int`) into which the words must be divided.
-If at least one of the words passed to the function `recouple` cannot be divided into $$n$$ groups that have the same length, an `AssertionError` must be raised with the message `invalid division`.
+If at least one of the words passed to the function `recouple` cannot be divided into $$n$$ groups that have the same length, an exception must be raised with the message `invalid division`.
 Otherwise, the function must return a sequence containing the $$n$$ new words (`str`) obtained when each of the $$m$$ given words is divided into $$n$$ groups that have the same length, and if each of the $$m$$ corresponding groups is merged into a new word.
 The type of the returned sequence (`list` or `tuple`) must correspond to the type of the sequence passed as a first argument to the function.
 
@@ -20,12 +20,12 @@ The type of the returned sequence (`list` or `tuple`) must correspond to the typ
 >>> divide('COMMUNED', 4)
 ['CO', 'MM', 'UN', 'ED']
 >>> divide('programming', 5)
-AssertionError: invalid division
+Exception: invalid division
 
 >>> recouple(['ACcoST', 'COmmIT', 'LAunCH', 'DEedED'], 3)
 ['ACCOLADE', 'communed', 'STITCHED']
 >>> recouple(('ACCOLADE', 'communed', 'STITCHED'), 4)
 ('ACcoST', 'COmmIT', 'LAunCH', 'DEedED')
 >>> recouple(['programming', 'computer', 'games'], 5)
-AssertionError: invalid division
+Exception: invalid division
 ```

tests/test_problem_statements.py

@@ -8,7 +8,7 @@
 
 
 @pytest.mark.parametrize("language", ["python", "kotlin", "java", "haskell"])
-def test_python_description(language: str):
+def test_small_descriptions(language: str):
     test_dir = Path(__file__).parent
     description_template = test_dir / "descriptions" / "example.md.jinja2"
     description_python = test_dir / "descriptions" / f"example.{language}.md"
@@ -18,7 +18,7 @@ def test_python_description(language: str):
         expected = dp.read()
     actual = process_problem_statement(template, language)
 
-    assert actual == expected
+    assert actual.strip() == expected.strip()
     assert f"This is {language}." in actual
 
 
@@ -130,38 +130,6 @@ def test_template_natural_type_name_nl(lang: str, tested_type: Any, expected: st
     assert instance == f"{expected}"
 
 
-@pytest.mark.parametrize(
-    ("lang", "prompt"),
-    [
-        ("python", ">>>"),
-        ("java", ">"),
-        ("c", ">"),
-        ("kotlin", ">"),
-        ("javascript", ">"),
-        ("haskell", ">"),
-    ],
-)
-def test_template_code_block_markdown(lang: str, prompt: str):
-    template = """```console?lang=tested
->>> random()
-5
-```"""
-    expected_stmt = (
-        "random "
-        if lang == "haskell"
-        else "Submission.random()"
-        if lang == "java"
-        else "random()"
-    )
-    expected_expr = "5"
-    instance = process_problem_statement(template, lang)
-    expected = f"""```console?lang={lang}&prompt={prompt}
-{prompt} {expected_stmt}
-{expected_expr}
-```"""
-    assert instance == expected
-
-
 @pytest.mark.parametrize(
     ("lang", "expected"),
     [
@@ -227,62 +195,6 @@ def test_template_failed_string():
         process_problem_statement(template, "java")
 
 
-def test_multiline_results():
-    template = """
-```console?lang=tested
->>> dots("paper.txt")
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
-    actual = process_problem_statement(template, "javascript")
-    expected = """
-```console?lang=javascript&prompt=>
-> dots("paper.txt")
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
-    assert actual == expected
-
-
-def test_multiline_statement():
-    template = """
-```console?lang=tested
->>> dots(
-...  "paper.txt"
-...  )
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
-    actual = process_problem_statement(template, "javascript")
-    expected = """
-```console?lang=javascript&prompt=>
-> dots("paper.txt")
-###..###...##..#..#.####.###..#....###.
-#..#.#..#.#..#.#.#..#....#..#.#....#..#
-#..#.#..#.#....##...###..###..#....#..#
-###..###..#....#.#..#....#..#.#....###.
-#.#..#....#..#.#.#..#....#..#.#....#.#.
-#..#.#.....##..#..#.#....###..####.#..#
-```
-"""
-    assert actual == expected
-
-
 def test_long_description():
     test_dir = Path(__file__).parent
     description_template = test_dir / "descriptions" / "recoupling.md.j2"