From 1d7f77fb297dec68094b46532fe7e6bfee8cb849 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 25 Jun 2024 14:25:02 +0200 Subject: [PATCH 1/2] Support arguments in language-specific oracles --- .editorconfig | 2 +- tested/languages/preparation.py | 48 +++++++--- tested/testsuite.py | 27 ++++-- .../echo-function/evaluation/Evaluator.cs | 6 ++ .../echo-function/evaluation/Evaluator.hs | 10 ++ .../echo-function/evaluation/Evaluator.java | 9 ++ .../echo-function/evaluation/Evaluator.kt | 9 ++ .../echo-function/evaluation/evaluator.c | 11 ++- .../echo-function/evaluation/evaluator.js | 11 +++ .../echo-function/evaluation/evaluator.py | 6 ++ .../evaluation/one-specific-argument.tson | 96 +++++++++++++++++++ tests/test_oracles_specific.py | 18 ++++ 12 files changed, 227 insertions(+), 26 deletions(-) create mode 100644 tests/exercises/echo-function/evaluation/one-specific-argument.tson diff --git a/.editorconfig b/.editorconfig index 9bcb33f9..cedfeae0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -16,7 +16,7 @@ ij_wrap_on_typing = false [*.nix] indent_size = 2 -[{*.json,*.yaml,*.yml}] +[{*.json,*.yaml,*.yml,*.tson}] indent_size = 2 ij_json_keep_blank_lines_in_code = 0 ij_json_keep_indents_on_empty_lines = false diff --git a/tested/languages/preparation.py b/tested/languages/preparation.py index 3521c6b6..0496bc7f 100644 --- a/tested/languages/preparation.py +++ b/tested/languages/preparation.py @@ -33,6 +33,7 @@ PropertyAssignment, SequenceType, Statement, + Value, VariableAssignment, VariableType, ) @@ -89,10 +90,10 @@ def input_statement(self, override: str | None = None) -> Statement: """ Get the input statement for the testcase. - This will return, depending on the command, either an expression which will + This will return, depending on the command, either an expression, which will pass the value to the correct handling function or the statement itself. - :param override: Optionally override the value argument. + :param override: Optionally, override the value argument. :return: The input statement. """ input_statement = ( @@ -285,24 +286,24 @@ def _create_handling_function( """ Create a function to handle the result of a return value or an exception. - There are two possibilities: - - There is a language-specific oracle. In that case, we wrap the value in - a function call to the oracle, and then send off the result. An example of - the result: + Either we need to evaluate the result in-process by using a language-specific + oracle or not. When using a language-specific oracle, we wrap the result in the + function call to the oracle and send the result of said evaluation to TESTed. + For example: - send_evaluated(evaluate(value)) + send_evaluated(oracle_function(result)) - - There is no language-specific oracle. In that case, we just send off the - value directly. An example of the result: + In the other case, we send the result directly to TESTed, where it will be + evaluated internally or with a custom check function. For example: - send_value(value) + send_value(result) :param bundle: The configuration bundle. :param send_evaluated: The name of the function that will handle sending the result of an evaluation. :param send_value: The name of the function that will handle sending the value. :param output: The oracle. - :return: A tuple containing the call and the name of the oracle if present. + :return: A tuple containing a function that can wrap a value and the name of the oracle if present. """ lang_config = bundle.language if isinstance(output, OracleOutputChannel) and isinstance( @@ -310,33 +311,50 @@ def _create_handling_function( ): evaluator = output.oracle.for_language(bundle.config.programming_language) evaluator_name = conventionalize_namespace(lang_config, evaluator.file.stem) + raw_args = output.oracle.get_arguments(bundle.config.programming_language) + evaluator_arguments = [] + for raw_arg in raw_args: + if isinstance(raw_arg, Value): + evaluator_arguments.append(prepare_argument(bundle, raw_arg)) + else: + assert isinstance(raw_arg, str) + the_arg = Identifier(raw_arg) + the_arg.is_raw = True + evaluator_arguments.append(prepare_argument(bundle, the_arg)) else: evaluator_name = None evaluator = None + evaluator_arguments = [] def generator(expression: Expression) -> Statement: if isinstance(output, OracleOutputChannel) and isinstance( output.oracle, LanguageSpecificOracle ): assert evaluator - arguments = [ + oracle_arguments = [ + # Next, pass the actual value. + prepare_expression(bundle, expression), + # Finally, prepare evaluator arguments. + *[prepare_argument(bundle, arg) for arg in evaluator_arguments], + ] + send_arguments = [ PreparedFunctionCall( type=FunctionType.FUNCTION, name=conventionalize_function(lang_config, evaluator.name), namespace=Identifier(evaluator_name), - arguments=[prepare_expression(bundle, expression)], + arguments=oracle_arguments, has_root_namespace=False, ) ] function_name = send_evaluated else: - arguments = [expression] + send_arguments = [expression] function_name = send_value internal = PreparedFunctionCall( type=FunctionType.FUNCTION, name=conventionalize_function(lang_config, function_name), - arguments=[prepare_argument(bundle, arg) for arg in arguments], + arguments=[prepare_argument(bundle, arg) for arg in send_arguments], has_root_namespace=False, ) return internal diff --git a/tested/testsuite.py b/tested/testsuite.py index 27564969..3c3606b9 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -172,11 +172,9 @@ class CustomCheckOracle: @define class LanguageSpecificOracle: """ - Evaluate the result with a custom check function written in a specific programming - language. Every programming language needs its own check function. - - While this is very powerful and allows you to test language-specific constructs, - there are a few caveats: + Evaluate the result with a custom check function written in the same programming + language as the submission. While this allows using language-specific constructs + not supported by TESTed, there are a few downsides: 1. The code is run alongside the user code. This means the user can potentially take control of the code. @@ -185,18 +183,29 @@ class LanguageSpecificOracle: 3. It is a lot of work. You need to return the correct values, since the judge needs to understand what the result was. - The code you must write should be a function that accepts the result of a user - expression. Note: this type of oracle is only supported when using function - calls. If you want to evaluate stdout, you should use the custom check oracle - instead. + The code you must write should be a function that accepts: + + 1. The value produced by the submission (the "actual" value) + 2. Optionally, any additional arguments provided in the test suite. + + The function must return a `BooleanEvaluationResult`, which, depending on the + programming language, takes the form of an instance of a class or a map/object. + See the examples in the tests for specifics. + + Note: this type of oracle is only supported when using function calls. If you + want to evaluate stdout, you should use the custom check oracle instead. """ functions: dict[SupportedLanguage, EvaluationFunction] = field() type: Literal["specific"] = "specific" + arguments: dict[SupportedLanguage, list[str | Value]] = field(factory=dict) def for_language(self, language: SupportedLanguage) -> EvaluationFunction: return self.functions[language] + def get_arguments(self, language: SupportedLanguage) -> list[str | Value]: + return self.arguments.get(language, []) + @functions.validator # type: ignore def validate_evaluator(self, _, value): """There should be at least one evaluator.""" diff --git a/tests/exercises/echo-function/evaluation/Evaluator.cs b/tests/exercises/echo-function/evaluation/Evaluator.cs index 8dc784d4..b19e90c1 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.cs +++ b/tests/exercises/echo-function/evaluation/Evaluator.cs @@ -7,4 +7,10 @@ public static EvaluationResult Evaluate(Object actual) { var messages = new List() {new Tested.Message("Hallo")}; return new EvaluationResult(correct, "correct", actual != null ? actual.ToString() : "", messages); } + + public static EvaluationResult EvaluateSum(Object actual, int sum) { + var correct = sum == 10; + var messages = new List() {new Tested.Message("Hallo")}; + return new EvaluationResult(correct, "correct", actual != null ? actual.ToString() : "", messages); + } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.hs b/tests/exercises/echo-function/evaluation/Evaluator.hs index e68ccc10..07f15eae 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.hs +++ b/tests/exercises/echo-function/evaluation/Evaluator.hs @@ -13,3 +13,13 @@ evaluate value = readableActual = Just value, messages = [message "Hallo"] } + +evaluateSum :: String -> Integer -> EvaluationResult +evaluateSum value the_sum = + let correct = the_sum == 10 + in evaluationResult { + result = correct, + readableExpected = Just "correct", + readableActual = Just value, + messages = [message "Hallo"] + } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.java b/tests/exercises/echo-function/evaluation/Evaluator.java index c4197043..69fd88e2 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.java +++ b/tests/exercises/echo-function/evaluation/Evaluator.java @@ -10,4 +10,13 @@ public static EvaluationResult evaluate(Object actual) { .withMessage(new EvaluationResult.Message("Hallo")) .build(); } + + public static EvaluationResult evaluateSum(Object actual, Integer sum) { + var correct = sum == 10; + return EvaluationResult.builder(correct) + .withReadableExpected("correct") + .withReadableActual(actual != null ? actual.toString() : "") + .withMessage(new EvaluationResult.Message("Hallo")) + .build(); + } } diff --git a/tests/exercises/echo-function/evaluation/Evaluator.kt b/tests/exercises/echo-function/evaluation/Evaluator.kt index 99cd36d6..b2e986bb 100644 --- a/tests/exercises/echo-function/evaluation/Evaluator.kt +++ b/tests/exercises/echo-function/evaluation/Evaluator.kt @@ -26,5 +26,14 @@ class Evaluator { .withMessage(EvaluationResult.Message("Hallo")) .build() } + + @JvmStatic + fun evaluateSum(actual: Any?, sum: Int): EvaluationResult { + return EvaluationResult.Builder(result = sum == 10, + readableExpected = actual.toString(), + readableActual = actual?.toString() ?: "") + .withMessage(EvaluationResult.Message("Hallo")) + .build() + } } } diff --git a/tests/exercises/echo-function/evaluation/evaluator.c b/tests/exercises/echo-function/evaluation/evaluator.c index 6a5908a7..14249512 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.c +++ b/tests/exercises/echo-function/evaluation/evaluator.c @@ -11,4 +11,13 @@ EvaluationResult* evaluate(char* actual) { r->readableActual = actual; r->messages[0] = create_message("Hallo", NULL, NULL); return r; -} \ No newline at end of file +} + +EvaluationResult* evaluate_sum(char* actual, int sum) { + EvaluationResult* r = create_result(1); + r->result = sum == 10; + r->readableExpected = "correct"; + r->readableActual = actual; + r->messages[0] = create_message("Hallo", NULL, NULL); + return r; +} diff --git a/tests/exercises/echo-function/evaluation/evaluator.js b/tests/exercises/echo-function/evaluation/evaluator.js index 10f8b7de..5621c3b2 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.js +++ b/tests/exercises/echo-function/evaluation/evaluator.js @@ -8,4 +8,15 @@ function evaluate(actual) { } } +function evaluateSum(actual, sum) { + const correct = sum == 10; + return { + "result": correct, + "readable_expected": "correct", + "readable_actual": actual.toString(), + "messages": [{"description": "Hallo", "format": "text"}] + } +} + exports.evaluate = evaluate; +exports.evaluateSum = evaluateSum; diff --git a/tests/exercises/echo-function/evaluation/evaluator.py b/tests/exercises/echo-function/evaluation/evaluator.py index 453be3e4..217a2cbf 100644 --- a/tests/exercises/echo-function/evaluation/evaluator.py +++ b/tests/exercises/echo-function/evaluation/evaluator.py @@ -22,3 +22,9 @@ def evaluate_value_dsl(context): def evaluate_runtime_crash(context): return len(context) / 0 + + +def evaluate_sum(actual, the_sum): + correct = the_sum == 10 + return EvaluationResult(correct, "correct", actual, [Message("Hallo")]) + diff --git a/tests/exercises/echo-function/evaluation/one-specific-argument.tson b/tests/exercises/echo-function/evaluation/one-specific-argument.tson new file mode 100644 index 00000000..c1029d6f --- /dev/null +++ b/tests/exercises/echo-function/evaluation/one-specific-argument.tson @@ -0,0 +1,96 @@ +{ + "tabs" : [ + { + "name" : "Tab", + "runs" : [ + { + "contexts" : [ + { + "testcases" : [ + { + "input" : { + "type" : "function", + "name" : "echo", + "arguments" : [ + { + "type" : "text", + "data" : "input-1" + } + ] + }, + "output" : { + "result" : { + "evaluator" : { + "type" : "specific", + "evaluators" : { + "python" : { + "file" : "evaluator.py", + "name": "evaluate_sum" + }, + "java" : { + "file" : "Evaluator.java", + "name": "evaluate_sum" + }, + "kotlin" : { + "file" : "Evaluator.kt", + "name": "evaluate_sum" + }, + "haskell" : { + "file" : "Evaluator.hs", + "name": "evaluate_sum" + }, + "runhaskell" : { + "file" : "Evaluator.hs", + "name": "evaluate_sum" + }, + "c" : { + "file" : "evaluator.c", + "name": "evaluate_sum" + }, + "javascript" : { + "file" : "evaluator.js", + "name": "evaluate_sum" + }, + "csharp" : { + "file" : "Evaluator.cs", + "name": "evaluate_sum" + } + }, + "arguments" : { + "python": [ + "5 + 5" + ], + "java": [ + "5 + 5" + ], + "kotlin": [ + "5 + 5" + ], + "haskell": [ + "5 + 5" + ], + "runhaskell": [ + "5 + 5" + ], + "c": [ + "5 + 5" + ], + "javascript": [ + "5 + 5" + ], + "csharp": [ + "5 + 5" + ] + } + } + } + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/tests/test_oracles_specific.py b/tests/test_oracles_specific.py index acb2fac5..a9ef7529 100644 --- a/tests/test_oracles_specific.py +++ b/tests/test_oracles_specific.py @@ -92,3 +92,21 @@ def test_specific_oracle_exception_runtime_exception( updates = assert_valid_output(result, pytestconfig) assert updates.find_status_enum() == ["wrong", "wrong"] assert len(updates.find_all("append-message")) >= 1 + + +@pytest.mark.parametrize("language", ALL_SPECIFIC_LANGUAGES) +def test_specific_oracle_return_specific_argument( + language: str, tmp_path: Path, pytestconfig: pytest.Config +): + conf = configuration( + pytestconfig, + "echo-function", + language, + tmp_path, + "one-specific-argument.tson", + "correct", + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] + assert len(updates.find_all("append-message")) == 1 From 7f2025e844ce49851f98708c4ac62d88015ebe03 Mon Sep 17 00:00:00 2001 From: Niko Strijbol Date: Tue, 25 Jun 2024 15:08:03 +0200 Subject: [PATCH 2/2] Add support to DSL for language-specific stuff --- tested/dsl/schema-strict.json | 166 +++++++++++++----- tested/dsl/schema.json | 159 ++++++++++++----- tested/dsl/translate_parser.py | 48 ++++- tested/languages/preparation.py | 11 +- tested/testsuite.py | 4 +- .../evaluation/one-specific-argument.tson | 32 ++-- .../evaluation/one-specific-argument.yaml | 48 +++++ tests/test_dsl_yaml.py | 81 +++++++++ tests/test_oracles_specific.py | 18 ++ 9 files changed, 445 insertions(+), 122 deletions(-) create mode 100644 tests/exercises/echo-function/evaluation/one-specific-argument.yaml diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index 6808880a..99c2f2a7 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -14,19 +14,27 @@ } ], "definitions" : { - "_rootObject": { - "type": "object", + "_rootObject" : { + "type" : "object", "oneOf" : [ { - "required" : ["tabs"], - "not": { - "required" : ["units"] + "required" : [ + "tabs" + ], + "not" : { + "required" : [ + "units" + ] } }, { - "required" : ["units"], - "not": { - "required" : ["tabs"] + "required" : [ + "units" + ], + "not" : { + "required" : [ + "tabs" + ] } } ], @@ -45,7 +53,7 @@ "tabs" : { "$ref" : "#/definitions/_tabList" }, - "units": { + "units" : { "$ref" : "#/definitions/_unitList" }, "language" : { @@ -59,9 +67,9 @@ } ] }, - "definitions": { - "description": "Define hashes to use elsewhere.", - "type": "object" + "definitions" : { + "description" : "Define hashes to use elsewhere.", + "type" : "object" } } }, @@ -100,9 +108,9 @@ "type" : "string", "description" : "The name of this tab." }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" } }, "oneOf" : [ @@ -149,9 +157,9 @@ "type" : "string", "description" : "The name of this tab." }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" } }, "oneOf" : [ @@ -177,28 +185,28 @@ } ] }, - "_contextList": { + "_contextList" : { "type" : "array", "minItems" : 1, "items" : { "$ref" : "#/definitions/context" } }, - "_caseList": { + "_caseList" : { "type" : "array", "minItems" : 1, "items" : { "$ref" : "#/definitions/case" } }, - "_testcaseList": { + "_testcaseList" : { "type" : "array", "minItems" : 1, "items" : { "$ref" : "#/definitions/testcase" } }, - "_scriptList": { + "_scriptList" : { "type" : "array", "minItems" : 1, "items" : { @@ -254,8 +262,8 @@ "description" : "An individual test for a statement or expression", "additionalProperties" : false, "properties" : { - "description": { - "$ref": "#/definitions/message" + "description" : { + "$ref" : "#/definitions/message" }, "stdin" : { "description" : "Stdin for this context", @@ -346,8 +354,8 @@ "type" : "object", "description" : "An individual test (script) for a statement or expression", "properties" : { - "description": { - "$ref": "#/definitions/message" + "description" : { + "$ref" : "#/definitions/message" }, "stdin" : { "description" : "Stdin for this context", @@ -455,7 +463,7 @@ } ] }, - "yamlValueOrPythonExpression": { + "yamlValueOrPythonExpression" : { "oneOf" : [ { "$ref" : "#/definitions/yamlValue" @@ -494,12 +502,14 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : ["data"], + "required" : [ + "data" + ], "properties" : { - "data": { + "data" : { "$ref" : "#/definitions/textualType" }, - "oracle": { + "oracle" : { "const" : "builtin" }, "config" : { @@ -510,9 +520,13 @@ { "type" : "object", "description" : "Custom oracle for text values.", - "required" : ["oracle", "file", "data"], + "required" : [ + "oracle", + "file", + "data" + ], "properties" : { - "oracle": { + "oracle" : { "const" : "custom_check" }, "file" : { @@ -535,7 +549,7 @@ } ] }, - "returnOutputChannel": { + "returnOutputChannel" : { "oneOf" : [ { "$ref" : "#/definitions/yamlValueOrPythonExpression" @@ -587,6 +601,63 @@ } } } + }, + { + "type" : "oracle", + "additionalProperties" : false, + "required" : [ + "oracle", + "functions" + ], + "properties" : { + "oracle" : { + "const" : "specific_check" + }, + "functions" : { + "minProperties" : 1, + "description" : "Language mapping of oracle functions.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "object", + "required" : [ + "file" + ], + "properties" : { + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + } + } + } + }, + "arguments" : { + "minProperties" : 1, + "description" : "Language mapping of oracle arguments.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + }, + "value" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } } ] }, @@ -605,23 +676,23 @@ "csharp" ] }, - "message": { - "oneOf": [ + "message" : { + "oneOf" : [ { - "type": "string", + "type" : "string", "description" : "A simple message to display." }, { - "type": "object", + "type" : "object", "required" : [ "description" ], "properties" : { - "description": { + "description" : { "type" : "string", "description" : "The message to display." }, - "format": { + "format" : { "type" : "string", "default" : "text", "description" : "The format of the message, either a programming language, 'text' or 'html'." @@ -647,9 +718,9 @@ "description" : "Ignore trailing whitespace", "type" : "boolean" }, - "normalizeTrailingNewlines": { - "description": "Normalize trailing newlines", - "type": "boolean" + "normalizeTrailingNewlines" : { + "description" : "Normalize trailing newlines", + "type" : "boolean" }, "roundTo" : { "description" : "The number of decimals to round at, when applying the rounding on floats", @@ -665,7 +736,7 @@ } } }, - "textualType": { + "textualType" : { "description" : "Simple textual value, converted to string.", "type" : [ "string", @@ -674,10 +745,13 @@ "boolean" ] }, - "yamlValue": { + "yamlValue" : { "description" : "A value represented as YAML.", - "not": { - "type": ["oracle", "expression"] + "not" : { + "type" : [ + "oracle", + "expression" + ] } } } diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index 1a4bc381..b7153193 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -14,19 +14,27 @@ } ], "definitions" : { - "_rootObject": { - "type": "object", + "_rootObject" : { + "type" : "object", "oneOf" : [ { - "required" : ["tabs"], - "not": { - "required" : ["units"] + "required" : [ + "tabs" + ], + "not" : { + "required" : [ + "units" + ] } }, { - "required" : ["units"], - "not": { - "required" : ["tabs"] + "required" : [ + "units" + ], + "not" : { + "required" : [ + "tabs" + ] } } ], @@ -45,7 +53,7 @@ "tabs" : { "$ref" : "#/definitions/_tabList" }, - "units": { + "units" : { "$ref" : "#/definitions/_unitList" }, "language" : { @@ -59,9 +67,9 @@ } ] }, - "definitions": { - "description": "Define hashes to use elsewhere.", - "type": "object" + "definitions" : { + "description" : "Define hashes to use elsewhere.", + "type" : "object" } } }, @@ -100,9 +108,9 @@ "type" : "string", "description" : "The name of this tab." }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" } }, "oneOf" : [ @@ -149,9 +157,9 @@ "type" : "string", "description" : "The name of this tab." }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" } }, "oneOf" : [ @@ -177,28 +185,28 @@ } ] }, - "_contextList": { + "_contextList" : { "type" : "array", "minItems" : 1, "items" : { "$ref" : "#/definitions/context" } }, - "_caseList": { + "_caseList" : { "type" : "array", "minItems" : 1, "items" : { "$ref" : "#/definitions/case" } }, - "_testcaseList": { + "_testcaseList" : { "type" : "array", "minItems" : 1, "items" : { "$ref" : "#/definitions/testcase" } }, - "_scriptList": { + "_scriptList" : { "type" : "array", "minItems" : 1, "items" : { @@ -254,8 +262,8 @@ "description" : "An individual test for a statement or expression", "additionalProperties" : false, "properties" : { - "description": { - "$ref": "#/definitions/message" + "description" : { + "$ref" : "#/definitions/message" }, "stdin" : { "description" : "Stdin for this context", @@ -346,8 +354,8 @@ "type" : "object", "description" : "An individual test (script) for a statement or expression", "properties" : { - "description": { - "$ref": "#/definitions/message" + "description" : { + "$ref" : "#/definitions/message" }, "stdin" : { "description" : "Stdin for this context", @@ -455,7 +463,7 @@ } ] }, - "yamlValueOrPythonExpression": { + "yamlValueOrPythonExpression" : { "oneOf" : [ { "$ref" : "#/definitions/yamlValue" @@ -494,12 +502,14 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : ["data"], + "required" : [ + "data" + ], "properties" : { - "data": { + "data" : { "$ref" : "#/definitions/textualType" }, - "oracle": { + "oracle" : { "const" : "builtin" }, "config" : { @@ -510,9 +520,13 @@ { "type" : "object", "description" : "Custom oracle for text values.", - "required" : ["oracle", "file", "data"], + "required" : [ + "oracle", + "file", + "data" + ], "properties" : { - "oracle": { + "oracle" : { "const" : "custom_check" }, "file" : { @@ -535,7 +549,7 @@ } ] }, - "returnOutputChannel": { + "returnOutputChannel" : { "oneOf" : [ { "$ref" : "#/definitions/yamlValueOrPythonExpression" @@ -587,6 +601,63 @@ } } } + }, + { + "type" : "object", + "additionalProperties" : false, + "required" : [ + "oracle", + "functions" + ], + "properties" : { + "oracle" : { + "const" : "specific_check" + }, + "functions" : { + "minProperties" : 1, + "description" : "Language mapping of oracle functions.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "object", + "required" : [ + "file" + ], + "properties" : { + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + } + } + } + }, + "arguments" : { + "minProperties" : 1, + "description" : "Language mapping of oracle arguments.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + }, + "value" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } } ] }, @@ -605,23 +676,23 @@ "csharp" ] }, - "message": { - "oneOf": [ + "message" : { + "oneOf" : [ { - "type": "string", + "type" : "string", "description" : "A simple message to display." }, { - "type": "object", + "type" : "object", "required" : [ "description" ], "properties" : { - "description": { + "description" : { "type" : "string", "description" : "The message to display." }, - "format": { + "format" : { "type" : "string", "default" : "text", "description" : "The format of the message, either a programming language, 'text' or 'html'." @@ -647,9 +718,9 @@ "description" : "Ignore trailing whitespace", "type" : "boolean" }, - "normalizeTrailingNewlines": { - "description": "Normalize trailing newlines", - "type": "boolean" + "normalizeTrailingNewlines" : { + "description" : "Normalize trailing newlines", + "type" : "boolean" }, "roundTo" : { "description" : "The number of decimals to round at, when applying the rounding on floats", @@ -665,7 +736,7 @@ } } }, - "textualType": { + "textualType" : { "description" : "Simple textual value, converted to string.", "type" : [ "string", @@ -674,7 +745,7 @@ "boolean" ] }, - "yamlValue": { + "yamlValue" : { "description" : "A value represented as YAML." } } diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 51e69b69..513caa18 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -32,7 +32,7 @@ resolve_to_basic, ) from tested.dodona import ExtendedMessage -from tested.dsl.ast_translator import parse_string +from tested.dsl.ast_translator import InvalidDslError, parse_string from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -56,6 +56,7 @@ GenericTextOracle, IgnoredChannel, LanguageLiterals, + LanguageSpecificOracle, MainInput, Output, Suite, @@ -368,6 +369,12 @@ def _convert_file(link_file: YamlDict) -> FileUrl: return FileUrl(name=link_file["name"], url=link_file["url"]) +def _convert_evaluation_function(stream: dict) -> EvaluationFunction: + return EvaluationFunction( + file=Path(stream["file"]), name=stream.get("name", "evaluate") + ) + + def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle: converted_args = [] for v in stream.get("arguments", []): @@ -375,13 +382,32 @@ def _convert_custom_check_oracle(stream: dict) -> CustomCheckOracle: assert isinstance(cv, Value) converted_args.append(cv) return CustomCheckOracle( - function=EvaluationFunction( - file=stream["file"], name=stream.get("name", "evaluate") - ), + function=_convert_evaluation_function(stream), arguments=converted_args, ) +def _convert_language_specific_oracle(stream: dict) -> LanguageSpecificOracle: + the_functions = dict() + for lang, a_function in stream["functions"].items(): + the_functions[SupportedLanguage(lang)] = _convert_evaluation_function( + a_function + ) + + the_args = dict() + for lang, args in stream.get("arguments", dict()).items(): + the_args[SupportedLanguage(lang)] = args + + if not set(the_args.keys()).issubset(the_functions.keys()): + raise InvalidDslError( + "Language-specific oracle found with arguments for non-oracle languages.\n\n" + f"You provided check functions for {the_functions.keys()}, but arguments for {the_args.keys()}.\n" + f"This means you have arguments for {the_args.keys() - the_functions.keys()} but no check function!" + ) + + return LanguageSpecificOracle(functions=the_functions, arguments=the_args) + + def _convert_text_output_channel(stream: YamlObject) -> TextOutputChannel: if isinstance(stream, str): data = _ensure_trailing_newline(stream) @@ -422,15 +448,25 @@ def _convert_yaml_value(stream: YamlObject) -> Value | None: def _convert_advanced_value_output_channel(stream: YamlObject) -> ValueOutputChannel: if isinstance(stream, ReturnOracle): return_object = stream - value = _convert_yaml_value(return_object["value"]) - assert isinstance(value, Value), "You must specify a value for a return oracle." if "oracle" not in return_object or return_object["oracle"] == "builtin": + value = _convert_yaml_value(return_object["value"]) + assert isinstance( + value, Value + ), "You must specify a value for a return oracle." return ValueOutputChannel(value=value) elif return_object["oracle"] == "custom_check": + value = _convert_yaml_value(return_object["value"]) + assert isinstance( + value, Value + ), "You must specify a value for a return oracle." return ValueOutputChannel( value=value, oracle=_convert_custom_check_oracle(return_object), ) + elif return_object["oracle"] == "specific_check": + return ValueOutputChannel( + oracle=_convert_language_specific_oracle(return_object) + ) raise TypeError(f"Unknown value oracle type: {return_object['oracle']}") else: yaml_value = _convert_yaml_value(stream) diff --git a/tested/languages/preparation.py b/tested/languages/preparation.py index 0496bc7f..ed496d80 100644 --- a/tested/languages/preparation.py +++ b/tested/languages/preparation.py @@ -33,7 +33,6 @@ PropertyAssignment, SequenceType, Statement, - Value, VariableAssignment, VariableType, ) @@ -314,13 +313,9 @@ def _create_handling_function( raw_args = output.oracle.get_arguments(bundle.config.programming_language) evaluator_arguments = [] for raw_arg in raw_args: - if isinstance(raw_arg, Value): - evaluator_arguments.append(prepare_argument(bundle, raw_arg)) - else: - assert isinstance(raw_arg, str) - the_arg = Identifier(raw_arg) - the_arg.is_raw = True - evaluator_arguments.append(prepare_argument(bundle, the_arg)) + the_arg = Identifier(raw_arg) + the_arg.is_raw = True + evaluator_arguments.append(prepare_argument(bundle, the_arg)) else: evaluator_name = None evaluator = None diff --git a/tested/testsuite.py b/tested/testsuite.py index 3c3606b9..0cf3a7f1 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -198,12 +198,12 @@ class LanguageSpecificOracle: functions: dict[SupportedLanguage, EvaluationFunction] = field() type: Literal["specific"] = "specific" - arguments: dict[SupportedLanguage, list[str | Value]] = field(factory=dict) + arguments: dict[SupportedLanguage, list[str]] = field(factory=dict) def for_language(self, language: SupportedLanguage) -> EvaluationFunction: return self.functions[language] - def get_arguments(self, language: SupportedLanguage) -> list[str | Value]: + def get_arguments(self, language: SupportedLanguage) -> list[str]: return self.arguments.get(language, []) @functions.validator # type: ignore diff --git a/tests/exercises/echo-function/evaluation/one-specific-argument.tson b/tests/exercises/echo-function/evaluation/one-specific-argument.tson index c1029d6f..12bbb4c1 100644 --- a/tests/exercises/echo-function/evaluation/one-specific-argument.tson +++ b/tests/exercises/echo-function/evaluation/one-specific-argument.tson @@ -25,60 +25,60 @@ "evaluators" : { "python" : { "file" : "evaluator.py", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "java" : { "file" : "Evaluator.java", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "kotlin" : { "file" : "Evaluator.kt", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "haskell" : { "file" : "Evaluator.hs", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "runhaskell" : { "file" : "Evaluator.hs", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "c" : { "file" : "evaluator.c", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "javascript" : { "file" : "evaluator.js", - "name": "evaluate_sum" + "name" : "evaluate_sum" }, "csharp" : { "file" : "Evaluator.cs", - "name": "evaluate_sum" + "name" : "evaluate_sum" } }, "arguments" : { - "python": [ + "python" : [ "5 + 5" ], - "java": [ + "java" : [ "5 + 5" ], - "kotlin": [ + "kotlin" : [ "5 + 5" ], - "haskell": [ + "haskell" : [ "5 + 5" ], - "runhaskell": [ + "runhaskell" : [ "5 + 5" ], - "c": [ + "c" : [ "5 + 5" ], - "javascript": [ + "javascript" : [ "5 + 5" ], - "csharp": [ + "csharp" : [ "5 + 5" ] } diff --git a/tests/exercises/echo-function/evaluation/one-specific-argument.yaml b/tests/exercises/echo-function/evaluation/one-specific-argument.yaml new file mode 100644 index 00000000..c098a1ab --- /dev/null +++ b/tests/exercises/echo-function/evaluation/one-specific-argument.yaml @@ -0,0 +1,48 @@ +- tab: "Tab" + testcases: + - expression: echo('input-1') + return: !oracle + oracle: specific_check + functions: + python: + file: evaluator.py + name: evaluate_sum + java: + file: Evaluator.java + name: evaluate_sum + kotlin: + file: Evaluator.kt + name: evaluate_sum + haskell: + file: Evaluator.hs + name: evaluate_sum + runhaskell: + file: Evaluator.hs + name: evaluate_sum + c: + file: evaluator.c + name: evaluate_sum + javascript: + file: evaluator.js + name: evaluate_sum + csharp: + file: Evaluator.cs + name: evaluate_sum + arguments: + python: + - 5 + 5 + java: + - 5 + 5 + kotlin: + - 5 + 5 + haskell: + - 5 + 5 + runhaskell: + - 5 + 5 + c: + - 5 + 5 + javascript: + - 5 + 5 + csharp: + - 5 + 5 + diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index 441fa095..497f1da4 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -35,6 +35,8 @@ GenericTextOracle, GenericValueOracle, LanguageLiterals, + LanguageSpecificOracle, + SupportedLanguage, TextOutputChannel, ValueOutputChannel, parse_test_suite, @@ -724,6 +726,85 @@ def test_value_custom_checks_correct(): ] +def test_value_specific_checks_correct(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test()' + return: !oracle + oracle: "specific_check" + functions: + python: + file: result.py + javascript: + file: result.js + arguments: + python: + - "yes" + javascript: + - "js" + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert len(tab.contexts) == 1 + testcases = tab.contexts[0].testcases + assert len(testcases) == 1 + test = testcases[0] + assert isinstance(test.input, FunctionCall) + assert isinstance(test.output.result, ValueOutputChannel) + assert isinstance(test.output.result.oracle, LanguageSpecificOracle) + oracle = test.output.result.oracle + assert oracle.functions[SupportedLanguage.PYTHON].name == "evaluate" + assert oracle.functions[SupportedLanguage.PYTHON].file == Path("result.py") + assert oracle.functions[SupportedLanguage.JAVASCRIPT].name == "evaluate" + assert oracle.functions[SupportedLanguage.JAVASCRIPT].file == Path("result.js") + assert oracle.arguments == { + SupportedLanguage.PYTHON: ["yes"], + SupportedLanguage.JAVASCRIPT: ["js"], + } + + +def test_value_specific_checks_missing_evaluators(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test()' + return: !oracle + oracle: "specific_check" + functions: + arguments: + python: + - "yes" + javascript: + - "js" + """ + with pytest.raises(Exception): + translate_to_test_suite(yaml_str) + + +def test_value_specific_checks_weird_arguments(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test()' + return: !oracle + oracle: "specific_check" + functions: + python: + file: "yes.py" + arguments: + javascript: + - "js" + """ + with pytest.raises(Exception): + translate_to_test_suite(yaml_str) + + def test_yaml_set_tag_is_supported(): yaml_str = """ - tab: 'Test' diff --git a/tests/test_oracles_specific.py b/tests/test_oracles_specific.py index a9ef7529..71c32fb0 100644 --- a/tests/test_oracles_specific.py +++ b/tests/test_oracles_specific.py @@ -110,3 +110,21 @@ def test_specific_oracle_return_specific_argument( updates = assert_valid_output(result, pytestconfig) assert updates.find_status_enum() == ["correct"] assert len(updates.find_all("append-message")) == 1 + + +@pytest.mark.parametrize("language", ALL_SPECIFIC_LANGUAGES) +def test_specific_oracle_return_specific_argument_dsl( + language: str, tmp_path: Path, pytestconfig: pytest.Config +): + conf = configuration( + pytestconfig, + "echo-function", + language, + tmp_path, + "one-specific-argument.yaml", + "correct", + ) + result = execute_config(conf) + updates = assert_valid_output(result, pytestconfig) + assert updates.find_status_enum() == ["correct"] + assert len(updates.find_all("append-message")) == 1