diff --git a/tested/__main__.py b/tested/__main__.py index 2f0c382d6..ba8779bb1 100644 --- a/tested/__main__.py +++ b/tested/__main__.py @@ -30,6 +30,13 @@ help="Include verbose logs. It is recommended to also use -o in this case.", action="store_true", ) + +parser.add_argument( + "-t", + "--translate", + type=str, + help="Specifies the language to translate translate the dsl to.", +) parser = parser.parse_args() if parser.verbose: @@ -42,4 +49,4 @@ configuration = read_config(parser.config) with smart_close(parser.output) as out: - run(configuration, out) + run(configuration, out, parser.translate) diff --git a/tested/configs.py b/tested/configs.py index 0cb0cc03a..ea7515396 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -8,6 +8,7 @@ from attrs import define, evolve, field +from tested.dodona import ExtendedMessage from tested.parsing import fallback_field, get_converter from tested.testsuite import ExecutionMode, Suite, SupportedLanguage from tested.utils import get_identifier, smart_close @@ -129,6 +130,7 @@ class Bundle: language: "Language" global_config: GlobalConfig out: IO + preprocessor_messages: list[ExtendedMessage] = [] @property def config(self) -> DodonaConfig: @@ -207,6 +209,7 @@ def create_bundle( output: IO, suite: Suite, language: str | None = None, + preprocessor_messages: list[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -216,6 +219,8 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. + :param preprocessor_messages: Indicator that the natural language translator + for the DSL key that was not defined in any translations map. :return: The configuration bundle. """ @@ -232,4 +237,12 @@ def create_bundle( suite=suite, ) lang_config = langs.get_language(global_config, language) - return Bundle(language=lang_config, global_config=global_config, out=output) + if preprocessor_messages is None: + preprocessor_messages = [] + + return Bundle( + language=lang_config, + global_config=global_config, + out=output, + preprocessor_messages=preprocessor_messages, + ) diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py new file mode 100644 index 000000000..f5949729f --- /dev/null +++ b/tested/dsl/dsl_errors.py @@ -0,0 +1,104 @@ +import sys +import textwrap + +import yaml +from jsonschema.exceptions import ValidationError + +from tested.dodona import ExtendedMessage, Permission + + +class InvalidYamlError(ValueError): + pass + + +class DslValidationError(ValueError): + pass + + +def convert_validation_error_to_group( + error: ValidationError, +) -> ExceptionGroup | Exception: + if not error.context and not error.cause: + if len(error.message) > 150: + message = error.message.replace(str(error.instance), "") + note = "With being: " + textwrap.shorten(str(error.instance), 500) + else: + message = error.message + note = None + converted = DslValidationError( + f"Validation error at {error.json_path}: " + message + ) + if note: + converted.add_note(note) + return converted + elif error.cause: + return error.cause + elif error.context: + causes = [convert_validation_error_to_group(x) for x in error.context] + message = f"Validation error at {error.json_path}, caused by a sub-exception." + return ExceptionGroup(message, causes) + else: + return error + + +def handle_dsl_validation_errors(errors: list): + if len(errors) == 1: + message = ( + "Validating the DSL resulted in an error. " + "The most specific sub-exception is often the culprit. " + ) + error = convert_validation_error_to_group(errors[0]) + if isinstance(error, ExceptionGroup): + raise ExceptionGroup(message, error.exceptions) + else: + raise DslValidationError(message + str(error)) from error + elif len(errors) > 1: + the_errors = [convert_validation_error_to_group(e) for e in errors] + message = "Validating the DSL resulted in some errors." + raise ExceptionGroup(message, the_errors) + + +def raise_yaml_error(yaml_stream: str, exc: yaml.MarkedYAMLError): + lines = yaml_stream.splitlines() + + if exc.problem_mark is None: + # There is no additional information, so what can we do? + raise exc + + sys.stderr.write( + textwrap.dedent( + f""" + YAML error while parsing test suite. This means there is a YAML syntax error. + + The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}: + + {lines[exc.problem_mark.line]} + {" " * exc.problem_mark.column + "^"} + + The error message was: + {exc.problem} {exc.context} + + The detailed exception is provided below. + You might also find help by validating your YAML file with a YAML validator.\n + """ + ) + ) + raise exc + + +def build_preprocessor_messages( + translations_missing_key: list[str], +) -> list[ExtendedMessage]: + """ + Build the preprocessor messages from the missing keys. + + :param translations_missing_key: The missing keys. + :return: The preprocessor messages. + """ + return [ + ExtendedMessage( + f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!", + permission=Permission.STAFF, + ) + for key in translations_missing_key + ] diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json new file mode 100644 index 000000000..167e0e147 --- /dev/null +++ b/tested/dsl/schema-strict-nat-translation.json @@ -0,0 +1,2073 @@ +{ + "$id" : "tested:dsl:schema7", + "$schema" : "http://json-schema.org/draft-07/schema#", + "title" : "TESTed-DSL", + "oneOf" : [ + { + "$ref" : "#/definitions/_rootObject" + }, + { + "$ref" : "#/definitions/_tabList" + }, + { + "$ref" : "#/definitions/_unitList" + } + ], + "definitions" : { + "_rootObject" : { + "type" : "object", + "oneOf" : [ + { + "required" : [ + "tabs" + ], + "not" : { + "required" : [ + "units" + ] + } + }, + { + "required" : [ + "units" + ], + "not" : { + "required" : [ + "tabs" + ] + } + } + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "namespace" : { + "type" : "string", + "description" : "Namespace of the submitted solution, in `snake_case`" + }, + "tabs" : { + "$ref" : "#/definitions/_tabList" + }, + "units" : { + "$ref" : "#/definitions/_unitList" + }, + "language" : { + "description" : "Indicate that all code is in a specific language.", + "oneOf" : [ + { + "$ref" : "#/definitions/programmingLanguage" + }, + { + "const" : "tested" + } + ] + }, + "translations" : { + "type" : "object", + "description": "Define translations in the global scope." + }, + "definitions" : { + "description" : "Define hashes to use elsewhere.", + "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" + } + } + }, + "_tabList" : { + "oneOf" : [ + { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/tab" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/tab" + } + } + } + } + } + ] + }, + "_unitList" : { + "oneOf" : [ + { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/unit" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/unit" + } + } + } + } + } + ] + }, + "tab" : { + "type" : "object", + "description" : "A tab in the test suite.", + "required" : [ + "tab" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "hidden" : { + "type" : "boolean", + "description" : "Defines if the unit/tab is hidden for the student or not" + }, + "tab" : { + "oneOf" : [ + { + "type" : "string" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string" + } + } + } + } + ], + "description" : "The name of this tab." + }, + "translations" : { + "type" : "object", + "description": "Define translations in the tab scope." + }, + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" + } + }, + "oneOf" : [ + { + "required" : [ + "contexts" + ], + "properties" : { + "contexts" : { + "$ref" : "#/definitions/_contextList" + } + } + }, + { + "required" : [ + "testcases" + ], + "properties" : { + "testcases" : { + "$ref" : "#/definitions/_testcaseList" + } + } + } + ] + }, + "unit" : { + "type" : "object", + "description" : "A unit in the test suite.", + "required" : [ + "unit" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "hidden" : { + "type" : "boolean", + "description" : "Defines if the unit/tab is hidden for the student or not" + }, + "unit" : { + "oneOf" : [ + { + "type" : "string", + "description" : "The name of this tab." + }, + { + "type" : "object", + "description" : "The name of this tab.", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string" + } + } + } + } + ], + "description" : "The name of this tab." + }, + "translations" : { + "type" : "object", + "description": "Define translations in the unit scope." + }, + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" + } + }, + "oneOf" : [ + { + "required" : [ + "cases" + ], + "properties" : { + "cases" : { + "$ref" : "#/definitions/_caseList" + } + } + }, + { + "required" : [ + "scripts" + ], + "properties" : { + "scripts" : { + "$ref" : "#/definitions/_scriptList" + } + } + } + ] + }, + "_contextList" : { + "oneOf" : [ + { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/context" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/context" + } + } + } + } + } + ] + }, + "_caseList" : { + "oneOf" : [ + { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/case" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/case" + } + } + } + } + } + ] + }, + "_testcaseList" : { + "oneOf" : [ + { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/testcase" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/testcase" + } + } + } + } + } + ] + }, + "_scriptList" : { + "oneOf" : [ + { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/script" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/script" + } + } + } + } + } + ] + }, + "context" : { + "type" : "object", + "description" : "A set of testcase in the same context.", + "required" : [ + "testcases" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "translations" : { + "type" : "object", + "description": "Define translations in the context scope." + }, + "context" : { + "type" : "string", + "description" : "Description of this context." + }, + "testcases" : { + "$ref" : "#/definitions/_testcaseList" + } + } + }, + "case" : { + "type" : "object", + "description" : "A test case.", + "required" : [ + "script" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "translations" : { + "type" : "object", + "description": "Define translations in the case scope." + }, + "context" : { + "type" : "string", + "description" : "Description of this context." + }, + "script" : { + "$ref" : "#/definitions/_scriptList" + } + } + }, + "testcase" : { + "type" : "object", + "description" : "An individual test for a statement or expression", + "additionalProperties" : false, + "properties" : { + "description" : { + "oneOf": [ + { + "$ref" : "#/definitions/message" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/message" + } + } + } + } + ] + }, + "stdin" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + ] + }, + "arguments" : { + "oneOf": [ + { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + } + ], + "description" : "Array of program call arguments" + }, + "statement" : { + "description" : "The statement to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "expression" : { + "description" : "The expression to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "exception" : { + "description" : "Expected exception message", + "oneOf" : [ + { + "$ref" : "#/definitions/exceptionChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/exceptionChannel" + } + } + } + } + ] + }, + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "return" : { + "description" : "Expected return value", + "oneOf" : [ + { + "$ref" : "#/definitions/returnOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/returnOutputChannel" + } + } + } + } + ] + + }, + "stderr" : { + "description" : "Expected output at stderr", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + } + } + ] + }, + "stdout" : { + "description" : "Expected output at stdout", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + } + } + ] + }, + "file": { + "description" : "Expected files generated by the submission.", + "oneOf" : [ + { + "$ref" : "#/definitions/fileOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/fileOutputChannel" + } + } + } + } + ] + }, + "exit_code" : { + "type" : "integer", + "description" : "Expected exit code for the run" + } + } + }, + "script" : { + "type" : "object", + "description" : "An individual test (script) for a statement or expression", + "properties" : { + "description" : { + "oneOf": [ + { + "$ref" : "#/definitions/message" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/message" + } + } + } + } + ] + }, + "stdin" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + ] + }, + "arguments" : { + "oneOf": [ + { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + } + ], + "description" : "Array of program call arguments" + }, + "statement" : { + "description" : "The statement to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "expression" : { + "description" : "The expression to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "exception" : { + "description" : "Expected exception message", + "oneOf" : [ + { + "$ref" : "#/definitions/exceptionChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/exceptionChannel" + } + } + } + } + ] + }, + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + } + } + ] + }, + "return" : { + "description" : "Expected return value", + "oneOf" : [ + { + "$ref" : "#/definitions/returnOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/returnOutputChannel" + } + } + } + } + ] + + }, + "stderr" : { + "description" : "Expected output at stderr", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + } + } + ] + }, + "stdout" : { + "description" : "Expected output at stdout", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + } + } + ] + }, + "file": { + "description" : "Expected files generated by the submission.", + "oneOf" : [ + { + "$ref" : "#/definitions/fileOutputChannel" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + } + } + ] + }, + "exit_code" : { + "type" : "integer", + "description" : "Expected exit code for the run" + } + } + }, + "expressionOrStatement" : { + "oneOf" : [ + { + "type" : "string", + "format" : "tested-dsl-expression", + "description" : "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "description" : "Programming-language-specific statement or expression.", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + ] + }, + "expressionOrStatementWithNatTranslation" : { + "oneOf" : [ + { + "type" : "string", + "format" : "tested-dsl-expression", + "description" : "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "description" : "Programming-language-specific statement or expression.", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/expressionOrStatement" + } + } + } + } + ] + }, + "yamlValueOrPythonExpression" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValue" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!expression" + }, + "value":{ + "type": "string", + "format" : "tested-dsl-expression", + "description" : "An expression in Python-syntax." + } + } + } + ] + }, + "file" : { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "name", + "url" + ], + "properties" : { + "name" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `description` folder of an exercise." + } + } + }, + "exceptionChannel" : { + "oneOf" : [ + { + "type" : "string", + "description" : "Message of the expected exception." + }, + { + "type" : "object", + "required" : [ + "types" + ], + "properties" : { + "message" : { + "oneOf" : [ + { + "type" : "string" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string" + } + } + } + } + ], + "description" : "Message of the expected exception." + }, + "types" : { + "minProperties" : 1, + "description" : "Language mapping of expected exception types.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string" + } + } + } + } + ] + }, + "textOutputChannel" : { + "anyOf" : [ + { + "$ref" : "#/definitions/textualType" + }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : [ + "data" + ], + "properties" : { + "data" : { + "oneOf" : [ + { + "$ref" : "#/definitions/textualType" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textualType" + } + } + } + } + ] + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "required" : [ + "oracle", + "file", + "data" + ], + "properties" : { + "data" : { + "oneOf" : [ + { + "$ref" : "#/definitions/textualType" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/textualType" + } + } + } + } + ] + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "returnOutputChannel" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!oracle" + }, + "value":{ + "type": "object", + "additionalProperties" : false, + "required" : [ + "value" + ], + "properties" : { + "oracle" : { + "const" : "builtin" + }, + "value" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + } + } + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!oracle" + }, + "value":{ + "type": "object", + "additionalProperties" : false, + "required" : [ + "value", + "oracle", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "value" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + } + } + ] + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + } + } + } + ], + "description" : "List of YAML (or tagged expression) values to use as arguments to the function." + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!oracle" + }, + "value":{ + "type": "object", + "additionalProperties" : false, + "required" : [ + "oracle", + "functions" + ], + "properties" : { + "oracle" : { + "const" : "specific_check" + }, + "functions" : { + "minProperties" : 1, + "description" : "Language mapping of oracle functions.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "object", + "required" : [ + "file" + ], + "properties" : { + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + } + } + } + }, + "arguments" : { + "oneOf" : [ + { + "minProperties" : 1, + "description" : "Language mapping of oracle arguments.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "minProperties" : 1, + "description" : "Language mapping of oracle arguments.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + } + } + ] + }, + "value" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + } + } + } + } + ] + }, + "programmingLanguage" : { + "type" : "string", + "description" : "One of the programming languages supported by TESTed.", + "enum" : [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + "message" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A simple message to display." + }, + { + "type" : "object", + "required" : [ + "description" + ], + "properties" : { + "description" : { + "oneOf": [ + { + "type" : "object", + "required": [ + "__tag__", + "value" + ], + "properties" : { + "__tag__": { + "type" : "string", + "description" : "The tag used in the yaml", + "const": "!natural_language" + }, + "value":{ + "type": "object", + "additionalProperties": { + "type" : "string" + } + } + } + }, + { + "type" : "string" + } + ], + "description" : "The message to display." + }, + "format" : { + "type" : "string", + "default" : "text", + "description" : "The format of the message, either a programming language, 'text' or 'html'." + } + } + } + ] + }, + "textConfigurationOptions" : { + "type" : "object", + "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties" : 1, + "properties" : { + "applyRounding" : { + "description" : "Apply rounding when comparing as float", + "type" : "boolean" + }, + "caseInsensitive" : { + "description" : "Ignore case when comparing strings", + "type" : "boolean" + }, + "ignoreWhitespace" : { + "description" : "Ignore trailing whitespace", + "type" : "boolean" + }, + "normalizeTrailingNewlines" : { + "description" : "Normalize trailing newlines", + "type" : "boolean" + }, + "roundTo" : { + "description" : "The number of decimals to round at, when applying the rounding on floats", + "type" : "integer" + }, + "tryFloatingPoint" : { + "description" : "Try comparing text as floating point numbers", + "type" : "boolean" + }, + "hideExpected" : { + "description" : "Hide the expected value in feedback (default: false), not recommended to use!", + "type" : "boolean" + } + } + }, + "fileConfigurationOptions": { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + }, + "textualType" : { + "description" : "Simple textual value, converted to string.", + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + "yamlValue" : { + "description" : "A value represented as YAML.", + "not" : { + "properties": { + "__tag__": { "type": "string" } + }, + "type": "object" + } + }, + "inheritableConfigObject": { + "type": "object", + "properties" : { + "stdout": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "stderr": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "file": { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + } + } +} diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 245b0cf7d..1e26a991e 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,6 +1,4 @@ import json -import sys -import textwrap from collections.abc import Callable from decimal import Decimal from pathlib import Path @@ -9,7 +7,6 @@ import yaml from attrs import define, evolve, field from jsonschema import TypeChecker -from jsonschema.exceptions import ValidationError from jsonschema.protocols import Validator from jsonschema.validators import extend as extend_validator from jsonschema.validators import validator_for @@ -33,6 +30,7 @@ ) from tested.dodona import ExtendedMessage from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string +from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( BooleanType, @@ -152,31 +150,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: try: return yaml.load(yaml_stream, loader) except yaml.MarkedYAMLError as exc: - lines = yaml_stream.splitlines() - - if exc.problem_mark is None: - # There is no additional information, so what can we do? - raise exc - - sys.stderr.write( - textwrap.dedent( - f""" - YAML error while parsing test suite. This means there is a YAML syntax error. - - The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}: - - {lines[exc.problem_mark.line]} - {" " * exc.problem_mark.column + "^"} - - The error message was: - {exc.problem} {exc.context} - - The detailed exception is provided below. - You might also find help by validating your YAML file with a YAML validator.\n - """ - ) - ) - raise exc + raise_yaml_error(yaml_stream, exc) def is_oracle(_checker: TypeChecker, instance: Any) -> bool: @@ -226,14 +200,6 @@ def validate_tested_dsl_expression(value: object) -> bool: return tested_validator(schema_object, format_checker=format_checker) -class DslValidationError(ValueError): - pass - - -class InvalidYamlError(ValueError): - pass - - @define(frozen=True) class DslContext: """ @@ -286,32 +252,6 @@ def merge_inheritable_with_specific_config( return recursive_dict_merge(inherited_options, specific_options) -def convert_validation_error_to_group( - error: ValidationError, -) -> ExceptionGroup | Exception: - if not error.context and not error.cause: - if len(error.message) > 150: - message = error.message.replace(str(error.instance), "") - note = "With being: " + textwrap.shorten(str(error.instance), 500) - else: - message = error.message - note = None - converted = DslValidationError( - f"Validation error at {error.json_path}: " + message - ) - if note: - converted.add_note(note) - return converted - elif error.cause: - return error.cause - elif error.context: - causes = [convert_validation_error_to_group(x) for x in error.context] - message = f"Validation error at {error.json_path}, caused by a sub-exception." - return ExceptionGroup(message, causes) - else: - return error - - def _validate_dsl(dsl_object: YamlObject): """ Validate a DSl object. @@ -320,20 +260,7 @@ def _validate_dsl(dsl_object: YamlObject): :return: True if valid, False otherwise. """ errors = list(load_schema_validator(dsl_object).iter_errors(dsl_object)) - if len(errors) == 1: - message = ( - "Validating the DSL resulted in an error. " - "The most specific sub-exception is often the culprit. " - ) - error = convert_validation_error_to_group(errors[0]) - if isinstance(error, ExceptionGroup): - raise ExceptionGroup(message, error.exceptions) - else: - raise DslValidationError(message + str(error)) from error - elif len(errors) > 1: - the_errors = [convert_validation_error_to_group(e) for e in errors] - message = "Validating the DSL resulted in some errors." - raise ExceptionGroup(message, the_errors) + handle_dsl_validation_errors(errors) def _tested_type_to_value(tested_type: TestedType) -> Value: diff --git a/tested/judge/core.py b/tested/judge/core.py index 24bdd3b31..811b21d79 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -116,6 +116,8 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) + if bundle.preprocessor_messages: + collector.add_messages(bundle.preprocessor_messages) max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() diff --git a/tested/main.py b/tested/main.py index b45490282..e72df435d 100644 --- a/tested/main.py +++ b/tested/main.py @@ -7,16 +7,19 @@ from tested.configs import DodonaConfig, create_bundle from tested.dsl import parse_dsl +from tested.nat_translation import apply_translations from tested.testsuite import parse_test_suite -def run(config: DodonaConfig, judge_output: IO): +def run(config: DodonaConfig, judge_output: IO, language: str | None = None): """ Run the TESTed judge. :param config: The configuration, as received from Dodona. :param judge_output: Where the judge output will be written to. + :param language: The language to use to translate the test-suite. """ + messages = [] try: with open(f"{config.resources}/{config.test_suite}", "r") as t: textual_suite = t.read() @@ -30,10 +33,13 @@ def run(config: DodonaConfig, judge_output: IO): _, ext = os.path.splitext(config.test_suite) is_yaml = ext.lower() in (".yaml", ".yml") if is_yaml: + if language: + textual_suite, messages = apply_translations(textual_suite, language) suite = parse_dsl(textual_suite) else: suite = parse_test_suite(textual_suite) - pack = create_bundle(config, judge_output, suite) + + pack = create_bundle(config, judge_output, suite, preprocessor_messages=messages) from .judge import judge judge(pack) diff --git a/tested/nat_translation.py b/tested/nat_translation.py new file mode 100644 index 000000000..65cfcfa59 --- /dev/null +++ b/tested/nat_translation.py @@ -0,0 +1,222 @@ +import json +import os +import sys +from pathlib import Path +from typing import Any, Type, cast + +import yaml +from jinja2 import Environment, TemplateSyntaxError, Undefined +from jsonschema.protocols import Validator +from jsonschema.validators import validator_for +from yaml.nodes import MappingNode, ScalarNode, SequenceNode + +from tested.dodona import ExtendedMessage +from tested.dsl.dsl_errors import ( + build_preprocessor_messages, + handle_dsl_validation_errors, + raise_yaml_error, +) + + +def validate_pre_dsl(yaml_object: Any): + """ + Validate a DSl object. + :param yaml_object: The object to validate. + """ + path_to_schema = Path(__file__).parent / "dsl/schema-strict-nat-translation.json" + with open(path_to_schema, "r") as schema_file: + schema_object = json.load(schema_file) + + validator: Type[Validator] = validator_for(schema_object) + schema_validator = validator(schema_object) + errors = list(schema_validator.iter_errors(yaml_object)) + handle_dsl_validation_errors(errors) + + +class CustomTagFormatDumper(yaml.SafeDumper): + + def represent_with_tag(self, tag, value): + if isinstance(value, dict): + return self.represent_mapping(tag, value) + elif isinstance(value, list): + return self.represent_sequence(tag, value) + else: + return self.represent_scalar(tag, value) + + @staticmethod + def custom_tag_format_representer(dumper, data): + """ + Will turn the given object back into YAML. + + :param dumper: The dumper to use. + :param data: The object to represent. + """ + if "__tag__" in data: + return dumper.represent_with_tag(data["__tag__"], data["value"]) + + return dumper.represent_mapping( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data + ) + + +def construct_custom_tag_format(loader, tag_suffix, node): + """ + This constructor will turn the given YAML into an object that can be used for translation. + + :param loader: The YAML loader. + :param tag_suffix: The tag that was found. + :param node: The node to construct. + """ + type2method = { + MappingNode: loader.construct_mapping, + ScalarNode: loader.construct_scalar, + SequenceNode: loader.construct_sequence, + } + + if not type(node) in type2method: + raise yaml.constructor.ConstructorError( + None, + None, + f"expected a mapping, scalar, or sequence node, but found {node.id}", + node.start_mark, + ) + + data = type2method[type(node)](node) + + return {"__tag__": tag_suffix, "value": data} + + +def translate_yaml( + data: Any, translations: dict, language: str, env: Environment +) -> Any: + """ + This function will translate the multilingual object. + + :param data: The object to translate. + :param translations: The merge of all found translations maps. + :param language: The language to translate to. + :param env: The Jinja-environment to use. + :return: The translated object. + """ + if isinstance(data, dict): + if "__tag__" in data and data["__tag__"] == "!natural_language": + value = data["value"] + assert language in value + return translate_yaml(value[language], translations, language, env) + + current_translations = data.pop("translations", {}) + for key, value in current_translations.items(): + assert language in value + current_translations[key] = value[language] + translations = {**translations, **current_translations} + + return { + key: translate_yaml(value, translations, language, env) + for key, value in data.items() + } + elif isinstance(data, list): + return [translate_yaml(item, translations, language, env) for item in data] + elif isinstance(data, str): + try: + return env.from_string(data).render(translations) + except TemplateSyntaxError: + return data + return data + + +def wrap_in_braces(value): + """ + This function will provide the ability to still keep the curly bracket around the + translated result. Example: {{ key | braces }} => {sleutel} and {{ key }} => sleutel. + """ + return f"{{{value}}}" + + +class TrackingUndefined(Undefined): + missing_keys = list() + + def __str__(self): + # Store the missing key name + TrackingUndefined.missing_keys.append(self._undefined_name) + # Return it in Jinja syntax to keep it in the template + return f"{{{{ {self._undefined_name} }}}}" + + __repr__ = __str__ + + +def create_enviroment() -> Environment: + enviroment = Environment(undefined=TrackingUndefined) + enviroment.filters["braces"] = wrap_in_braces + return enviroment + + +def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str): + file_name = yaml_path.name + split_name = file_name.split(".") + path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml" + with open(path_to_new_yaml, "w", encoding="utf-8") as yaml_file: + yaml_file.write(yaml_string) + + +def convert_to_yaml(translated_data: Any) -> str: + CustomTagFormatDumper.add_representer( + dict, CustomTagFormatDumper.custom_tag_format_representer + ) + return yaml.dump( + translated_data, + Dumper=CustomTagFormatDumper, + allow_unicode=True, + sort_keys=False, + ) + + +def parse_yaml(yaml_stream: str) -> Any: + """ + Parse a string or stream to YAML. + """ + loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader) + yaml.add_multi_constructor("", construct_custom_tag_format, loader) + + try: + return yaml.load(yaml_stream, loader) + except yaml.MarkedYAMLError as exc: + raise_yaml_error(yaml_stream, exc) + + +def apply_translations( + yaml_stream: str, language: str +) -> tuple[str, list[ExtendedMessage]]: + parsed_yaml = parse_yaml(yaml_stream) + validate_pre_dsl(parsed_yaml) + + enviroment = create_enviroment() + translated_data = translate_yaml(parsed_yaml, {}, language, enviroment) + + missing_keys = TrackingUndefined.missing_keys + messages = build_preprocessor_messages(missing_keys) + translated_yaml_string = convert_to_yaml(translated_data) + return translated_yaml_string, messages + + +def translate_file(path: Path, language: str) -> tuple[str, list[ExtendedMessage]]: + try: + with open(path, "r") as stream: + yaml_stream = stream.read() + except FileNotFoundError as e: + print("The test suite was not found. Check your exercise's config.json file.") + print( + "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise." + ) + raise e + _, ext = os.path.splitext(path) + assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}." + translated_yaml_string, messages = apply_translations(yaml_stream, language) + generate_new_yaml(path, translated_yaml_string, language) + return translated_yaml_string, messages + + +if __name__ == "__main__": + n = len(sys.argv) + assert n > 1, "Expected atleast two argument (path to yaml file and language)." + + translate_file(Path(sys.argv[1]), sys.argv[2]) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py new file mode 100644 index 000000000..34bb52270 --- /dev/null +++ b/tests/test_preprocess_dsl.py @@ -0,0 +1,642 @@ +from pathlib import Path + +import yaml +from pytest_mock import MockerFixture + +import tested +from tested.dodona import ExtendedMessage, Permission +from tested.dsl.translate_parser import ( + ExpressionString, + ReturnOracle, + _parse_yaml, + _validate_dsl, +) +from tested.nat_translation import ( + convert_to_yaml, + create_enviroment, + parse_yaml, + translate_file, + translate_yaml, + validate_pre_dsl, +) + + +def validate_natural_translate(yaml_str: str, translated_yaml_str: str): + enviroment = create_enviroment() + yaml_object = parse_yaml(yaml_str) + translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment) + translated_yaml = convert_to_yaml(translated_dsl) + assert translated_yaml.strip() == translated_yaml_str + + +def test_files_and_descriptions(): + yaml_str = """ +translations: + animal: + en: "animals" + nl: "dieren" + result: + en: "results" + nl: "resultaten" +tabs: + - tab: "{{animal}}" + contexts: + - testcases: + - statement: !natural_language + en: '{{result}}: dict = Trying("file.txt")' + nl: '{{result}}: dict = Proberen("fileNL.txt")' + - expression: !natural_language + en: 'count_words({{result}})' + nl: 'tel_woorden({{result}})' + return: !natural_language + en: 'The {{result}} is 10' + nl: 'Het {{result}} is 10' + description: !natural_language + en: "Ten" + nl: "Tien" + files: !natural_language + en: + - name: "file.txt" + url: "media/workdir/file.txt" + nl: + - name: "fileNL.txt" + url: "media/workdir/fileNL.txt" + translations: + result: + en: "results_context" + nl: "resultaten_context" + """.strip() + translated_yaml_str = """ +tabs: +- tab: animals + contexts: + - testcases: + - statement: 'results_context: dict = Trying("file.txt")' + - expression: count_words(results_context) + return: The results_context is 10 + description: Ten + files: + - name: file.txt + url: media/workdir/file.txt +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_return(): + yaml_str = """ +translations: + animal: + en: "animals" + nl: "dieren" + result: + en: "results" + nl: "resultaten" +tabs: + - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}" + translations: + animal: + en: "animal_tab" + nl: "dier_tab" + contexts: + - testcases: + - expression: !natural_language + en: "count" + nl: "tellen" + return: !natural_language + en: !expression 'count' + nl: !expression 'tellen' + - expression: 'ok(10)' + return: !oracle + value: !natural_language + en: "The {{result}} 10 is OK!" + nl: "Het {{result}} 10 is OK!" + oracle: "custom_check" + file: "test.py" + name: "evaluate_test" + arguments: !natural_language + en: ["The value", "is OK!", "is not OK!"] + nl: ["Het {{result}}", "is OK!", "is niet OK!"] + """.strip() + translated_yaml_str = """ +tabs: +- tab: '{animal_tab}_{results}' + contexts: + - testcases: + - expression: count + return: !expression 'count' + - expression: ok(10) + return: !oracle + value: The results 10 is OK! + oracle: custom_check + file: test.py + name: evaluate_test + arguments: + - The value + - is OK! + - is not OK! +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_nat_lang_and_prog_lang_combination(): + yaml_str = """ +translations: + animal: + en: "animals" + nl: "dieren" +tabs: + - tab: '{{animal}}' + testcases: + - expression: !natural_language + en: "tests(11)" + nl: "testen(11)" + return: 11 + - expression: + javascript: "{{animal}}_javascript(1 + 1)" + typescript: "{{animal}}_typescript(1 + 1)" + java: "Submission.{{animal}}_java(1 + 1)" + python: !natural_language + en: "{{animal}}_python_en(1 + 1)" + nl: "{{animal}}_python_nl(1 + 1)" + return: 2 +""".strip() + translated_yaml_str = """ +tabs: +- tab: animals + testcases: + - expression: tests(11) + return: 11 + - expression: + javascript: animals_javascript(1 + 1) + typescript: animals_typescript(1 + 1) + java: Submission.animals_java(1 + 1) + python: animals_python_en(1 + 1) + return: 2 +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_format_expression(): + yaml_str = """ +translations: + select: + en: "select" + nl: "selecteer" +tabs: + - tab: 'test' + testcases: + - expression: "{{select}}('a', {'a': 1, 'b': 2})" + return: 1 +""".strip() + translated_yaml_str = """ +tabs: +- tab: test + testcases: + - expression: 'select(''a'', {''a'': 1, ''b'': 2})' + return: 1 +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_natural_translate_context(): + yaml_str = """ +translations: + result: + en: "results" + nl: "resultaten" +tabs: + - tab: "task" + contexts: !natural_language + en: + - testcases: + - statement: '{{result}} = Trying(10)' + - expression: 'count_words({{result}})' + return: 'The {{result}} is 10' + - expression: "count" + return: !expression 'count' + nl: + - testcases: + - statement: '{{result}} = Proberen(10)' + - expression: 'tel_woorden({{result}})' + return: 'Het {{result}} is 10' + - expression: "tellen" + return: !expression 'tellen' +""".strip() + translated_yaml_str = """ +tabs: +- tab: task + contexts: + - testcases: + - statement: results = Trying(10) + - expression: count_words(results) + return: The results is 10 + - expression: count + return: !expression 'count' +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_natural_translate_testcases_in_context(): + yaml_str = """ +translations: + result: + en: "results" + nl: "resultaten" +tabs: + - tab: "task2" + contexts: + - testcases: !natural_language + en: + - statement: '{{result}} = Trying(10)' + - expression: 'count_words({{result}})' + return: 'The {{result}} is 10' + - expression: "count" + return: !expression 'count' + nl: + - statement: '{{result}} = Proberen(10)' + - expression: 'tel_woorden({{result}})' + return: 'Het {{result}} is 10' + - expression: "tellen" + return: !expression 'tellen' +""".strip() + translated_yaml_str = """ +tabs: +- tab: task2 + contexts: + - testcases: + - statement: results = Trying(10) + - expression: count_words(results) + return: The results is 10 + - expression: count + return: !expression 'count' +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_natural_translate_testcases(): + yaml_str = """ +translations: + result: + en: "results" + nl: "resultaten" +tabs: + - tab: "task3" + testcases: !natural_language + en: + - statement: '{{result}} = Trying(10)' + - expression: 'count_words({{result}})' + return: 'The {{result}} is 10' + - expression: "count" + return: !expression 'count' + nl: + - statement: '{{result}} = Proberen(10)' + - expression: 'tel_woorden({{result}})' + return: 'Het {{result}} is 10' + - expression: "tellen" + return: !expression 'tellen' +""".strip() + translated_yaml_str = """ +tabs: +- tab: task3 + testcases: + - statement: results = Trying(10) + - expression: count_words(results) + return: The results is 10 + - expression: count + return: !expression 'count' +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_natural_translate_io_test(): + yaml_str = """ +units: + - unit: !natural_language + en: "Arguments" + nl: "Argumenten" + translations: + User: + en: "user" + nl: "gebruiker" + cases: + - script: + - stdin: !natural_language + en: "User_{{User}}" + nl: "Gebruiker_{{User}}" + arguments: !natural_language + en: [ "input_{{User}}", "output_{{User}}" ] + nl: [ "invoer_{{User}}", "uitvoer_{{User}}" ] + stdout: !natural_language + en: "Hi {{User}}" + nl: "Hallo {{User}}" + stderr: !natural_language + en: "Nothing to see here {{User}}" + nl: "Hier is niets te zien {{User}}" + exception: !natural_language + en: "Does not look good" + nl: "Ziet er niet goed uit" +""".strip() + translated_yaml_str = """ +units: +- unit: Arguments + cases: + - script: + - stdin: User_user + arguments: + - input_user + - output_user + stdout: Hi user + stderr: Nothing to see here user + exception: Does not look good +""".strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_validation(): + yaml_str = """ +translations: + animal: + en: "animals" + nl: "dieren" + result: + en: "results" + nl: "resultaten" + elf: + en: "eleven" + nl: "elf" + select: + en: "select" + nl: "selecteer" +tabs: + - tab: "{{animal}}" + contexts: + - testcases: + - expression: !natural_language + en: "count" + nl: "tellen" + return: !natural_language + en: !expression 'count' + nl: !expression 'tellen' + - expression: 'ok(10)' + return: !oracle + value: !natural_language + en: "The {{result}} 10 is OK!" + nl: "Het {{result}} 10 is OK!" + oracle: "custom_check" + file: "test.py" + name: "evaluate_test" + arguments: !natural_language + en: ["The value", "is OK!", "is not OK!"] + nl: ["Het {{result}}", "is OK!", "is niet OK!"] + """ + yaml_object = parse_yaml(yaml_str) + validate_pre_dsl(yaml_object) + + enviroment = create_enviroment() + translated_data = translate_yaml(yaml_object, {}, "en", enviroment) + translated_yaml_string = convert_to_yaml(translated_data) + _validate_dsl(_parse_yaml(translated_yaml_string)) + + +def test_wrong_natural_translation_suite(): + yaml_str = """ +tabs: +- tab: animals + testcases: + - expression: tests(11) + return: !exp 11 + - expression: + javascript: animals_javascript(1 + 1) + typescript: animals_typescript(1 + 1) + java: Submission.animals_java(1 + 1) + python: + en: animals_python_en(1 + 1) + nl: animals_python_nl(1 + 1) + return: 2 + """.strip() + parsed_yaml = parse_yaml(yaml_str) + try: + validate_pre_dsl(parsed_yaml) + except ExceptionGroup: + print("As expected") + else: + assert False, "Expected ExceptionGroup error" + + +def test_run_is_correct(mocker: MockerFixture): + s = mocker.spy(tested.nat_translation, name="generate_new_yaml") # type: ignore[reportAttributeAccessIssue] + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in [ + """ +tabs: +- tab: task3 + testcases: + - statement: !natural_language + nl: resultaten = Proberen(10) + en: results = Tries(10) + - expression: !natural_language + nl: tel_woorden(resultaten) + en: count_words(results) + return: !natural_language + nl: Het resultaat is 10 + en: The result is 10""" + ] + ] + mock_files.append(mocker.mock_open(read_data="{}").return_value) + mock_files.append(mocker.mock_open().return_value) + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + + translated_yaml, _ = translate_file(Path("suite.yaml"), "en") + yaml_object = parse_yaml(translated_yaml) + + assert s.call_count == 1 + assert isinstance(yaml_object, dict) + tabs = yaml_object["tabs"] + assert isinstance(tabs, list) + assert tabs[0]["testcases"][0] == {"statement": "results = Tries(10)"} + assert tabs[0]["testcases"][1] == { + "expression": "count_words(results)", + "return": "The result is 10", + } + + # Check if the file was opened for writing + mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8") + + +def test_key_not_found(mocker: MockerFixture): + s = mocker.spy( + tested.nat_translation, name="generate_new_yaml" # type: ignore[reportAttributeAccessIssue] + ) + + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in [ + """ +tabs: +- tab: task3 + testcases: + - statement: !natural_language + nl: resultaten = Proberen({{ten}}) + en: results = Tries({{ten}}) + - expression: !natural_language + nl: tel_woorden(resultaten) + en: count_words(results) + return: !natural_language + nl: Het resultaat is 10 + en: The result is 10""" + ] + ] + mock_files.append(mocker.mock_open(read_data="{}").return_value) + mock_files.append(mocker.mock_open().return_value) + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + + _, messages = translate_file(Path("suite.yaml"), "en") + + assert messages + assert s.call_count == 1 + assert isinstance(messages[0], ExtendedMessage) + assert ( + messages[0].description + == "The natural translator found the key ten, that was not defined in the corresponding translations maps!" + ) + assert messages[0].permission == Permission.STAFF + + # Check if the file was opened for writing + mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8") + + +def test_parsing_failed(): + yaml_str = """ +tabs: +- tab: animals + testcases: + - expression: tests(11) + return: 11 + expression: calculator(1 + 1) + - return: 2 + """.strip() + try: + parse_yaml(yaml_str) + except yaml.MarkedYAMLError: + print("As expected") + else: + assert False, "Expected MarkedYAMLError error" + + +def test_to_yaml_object(): + yaml_str = """ +translations: + animal: + en: "animals" + nl: "dieren" + result: + en: "results" + nl: "resultaten" + elf: + en: "eleven" + nl: "elf" + select: + en: "select" + nl: "selecteer" +tabs: + - tab: "{{animal}}" + contexts: + - testcases: + - expression: !natural_language + en: "count" + nl: "tellen" + return: !natural_language + en: !expression 'count' + nl: !expression 'tellen' + - expression: 'ok(10)' + return: !oracle + value: !natural_language + en: "The {{result}} 10 is OK!" + nl: "Het {{result}} 10 is OK!" + oracle: "custom_check" + file: "test.py" + name: "evaluate_test" + arguments: !natural_language + en: ["The value", "is OK!", "is not OK!"] + nl: ["Het {{result}}", "is OK!", "is niet OK!"] + """ + + environment = create_enviroment() + parsed_yaml = parse_yaml(yaml_str) + translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment) + translated_yaml_string = convert_to_yaml(translated_dsl) + yaml_object = _parse_yaml(translated_yaml_string) + assert isinstance(yaml_object, dict) + tabs = yaml_object["tabs"] + assert isinstance(tabs, list) + context = tabs[0]["contexts"][0] + assert isinstance(context, dict) + testcase = context["testcases"][0] + assert isinstance(testcase, dict) + assert isinstance(testcase["return"], ExpressionString) + + testcase = context["testcases"][1] + assert isinstance(testcase, dict) + assert isinstance(testcase["return"], ReturnOracle) + + +def test_dumper(): + translated_dsl = {"__tag__": "!tag", "value": [1, 2, 3]} + yaml_str = convert_to_yaml(translated_dsl).strip() + expected = """ +!tag +- 1 +- 2 +- 3 + """.strip() + assert yaml_str == expected + + translated_dsl = {"__tag__": "!tag", "value": {"key1": "value1", "key2": "value2"}} + yaml_str = convert_to_yaml(translated_dsl).strip() + expected = """ +!tag +key1: value1 +key2: value2 + """.strip() + assert yaml_str == expected + + translated_dsl = {"__tag__": "!tag", "value": "value"} + yaml_str = convert_to_yaml(translated_dsl).strip() + expected = "!tag 'value'" + assert yaml_str == expected + + +def test_run_is_correct_when_no_file(): + try: + translate_file(Path("suite.yaml"), "en") + except FileNotFoundError: + print("As expected") + else: + assert False, "Expected FileNotFoundError error" + + +def test_template_syntax_error(): + yaml_str = """ +translations: + works: + en: "works" + nl: "werkt" +tabs: +- tab: animals + testcases: + - expression: tests(11) + return: 11{%{{works}} +""".strip() + translated_yml = """ +tabs: +- tab: animals + testcases: + - expression: tests(11) + return: 11{%{{works}} +""".strip() + validate_natural_translate(yaml_str, translated_yml)