Fix bug in value oracle

There was an issue with the check on lists of different lengths.
dodona-edu · Jan 31, 2024 · c7caef4 · c7caef4
1 parent f8b7539
commit c7caef4
Show file tree

Hide file tree

Showing 2 changed files with 126 additions and 0 deletions.
diff --git a/tested/oracles/value.py b/tested/oracles/value.py
@@ -207,6 +207,23 @@ def _check_data_type(
             )
             prepared_elements.append(prepared_element)
             valid = valid and element_valid
+
+        # If one of the lists is not the same length as the other, we must do more.
+        if len(prepared_elements) != len(expected_elements):
+            valid = False
+            if len(expected_elements) > len(prepared_elements):
+                longest_list = expected_elements
+            else:
+                assert len(actual_elements) > len(prepared_elements)
+                longest_list = actual_elements
+
+            for remaining_element in longest_list[len(longest_list) :]:
+                assert remaining_element is None or isinstance(remaining_element, Value)
+                _, prepared_element = _check_data_type(
+                    bundle, remaining_element, remaining_element
+                )
+                prepared_elements.append(prepared_element)
+
         prepared_expected.data = prepared_elements
     elif isinstance(prepared_expected, ObjectType):
         expected_elements = prepared_expected.data

diff --git a/tests/test_oracles.py b/tests/test_oracles.py
@@ -552,3 +552,112 @@ def test_too_many_object_values_dont_crash(tmp_path: Path, pytestconfig):
         get_converter().dumps(actual_value),
     )
     assert result.result.enum == Status.WRONG
+
+
+def test_values_different_lengths_are_detected_empty_actual(
+    tmp_path: Path, pytestconfig
+):
+    channel = ValueOutputChannel(
+        value=SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+            ],
+        )
+    )
+    actual_value = get_converter().dumps(
+        SequenceType(type=BasicSequenceTypes.SEQUENCE, data=[])
+    )
+    config = oracle_config(tmp_path, pytestconfig, language="python")
+    result = evaluate_value(config, channel, actual_value)
+    assert result.result.enum == Status.WRONG
+
+
+def test_values_different_lengths_are_detected_empty_expected(
+    tmp_path: Path, pytestconfig
+):
+    channel = ValueOutputChannel(
+        value=SequenceType(type=BasicSequenceTypes.SEQUENCE, data=[])
+    )
+    actual_value = get_converter().dumps(
+        SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+            ],
+        )
+    )
+    config = oracle_config(tmp_path, pytestconfig, language="python")
+    result = evaluate_value(config, channel, actual_value)
+    assert result.result.enum == Status.WRONG
+
+
+def test_values_different_lengths_are_detected_different(tmp_path: Path, pytestconfig):
+    channel = ValueOutputChannel(
+        value=SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+            ],
+        )
+    )
+    actual_value = get_converter().dumps(
+        SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None)],
+        )
+    )
+    config = oracle_config(tmp_path, pytestconfig, language="python")
+    result = evaluate_value(config, channel, actual_value)
+    assert result.result.enum == Status.WRONG
+
+
+def test_values_same_lengths_are_detected_different(tmp_path: Path, pytestconfig):
+    channel = ValueOutputChannel(
+        value=SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+            ],
+        )
+    )
+    actual_value = get_converter().dumps(
+        SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="A", diagnostic=None),
+            ],
+        )
+    )
+    config = oracle_config(tmp_path, pytestconfig, language="python")
+    result = evaluate_value(config, channel, actual_value)
+    assert result.result.enum == Status.WRONG
+
+
+def test_values_identical_list_is_detected(tmp_path: Path, pytestconfig):
+    channel = ValueOutputChannel(
+        value=SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+            ],
+        )
+    )
+    actual_value = get_converter().dumps(
+        SequenceType(
+            type=BasicSequenceTypes.SEQUENCE,
+            data=[
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+                StringType(type=BasicStringTypes.TEXT, data="S", diagnostic=None),
+            ],
+        )
+    )
+    config = oracle_config(tmp_path, pytestconfig, language="python")
+    result = evaluate_value(config, channel, actual_value)
+    assert result.result.enum == Status.CORRECT