diff --git a/tested/configs.py b/tested/configs.py index 0cb0cc03a..5e58010fd 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -8,6 +8,7 @@ from attrs import define, evolve, field +from tested.dodona import ExtendedMessage from tested.parsing import fallback_field, get_converter from tested.testsuite import ExecutionMode, Suite, SupportedLanguage from tested.utils import get_identifier, smart_close @@ -129,6 +130,7 @@ class Bundle: language: "Language" global_config: GlobalConfig out: IO + messages: set[ExtendedMessage] = set() @property def config(self) -> DodonaConfig: @@ -207,6 +209,7 @@ def create_bundle( output: IO, suite: Suite, language: str | None = None, + messages: set[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -216,7 +219,7 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. - + :param messages: Messages generated out of the translate parser. :return: The configuration bundle. """ import tested.languages as langs @@ -232,4 +235,10 @@ def create_bundle( suite=suite, ) lang_config = langs.get_language(global_config, language) - return Bundle(language=lang_config, global_config=global_config, out=output) + + if messages is None: + messages = set() + + return Bundle( + language=lang_config, global_config=global_config, out=output, messages=messages + ) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 5690f12c4..2e766ac68 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,7 +83,7 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl = parse_dsl(rendered_dsl) + parsed_dsl = parse_dsl(rendered_dsl).data # Get all actual tests tests = [] @@ -95,7 +95,7 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: resulting_lines = [] prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">") for testcase in tests: - stmt_message, _ = get_readable_input(self.bundle, testcase) + stmt_message = get_readable_input(self.bundle, testcase) resulting_lines.append(f"{prompt} {stmt_message.description}") output_lines = get_expected_output(self.bundle, testcase) resulting_lines.extend(output_lines) diff --git a/tested/dodona.py b/tested/dodona.py index 90e5f7683..3ece258a2 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -27,7 +27,7 @@ class Permission(StrEnum): ZEUS = auto() -@define +@define(frozen=True) class ExtendedMessage: description: str format: str = "text" @@ -36,10 +36,11 @@ class ExtendedMessage: @define class Metadata: - """Currently only used for the Python tutor""" + """Currently used for the Python tutor and rendering files in Dodona.""" statements: str | None stdin: str | None + files: list[dict[str, str]] | None Message = ExtendedMessage | str diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index d9e3f4074..2aafb2548 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -39,7 +39,14 @@ } ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +104,15 @@ "tab" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +164,15 @@ "unit" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +252,15 @@ "testcases" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +282,15 @@ "script" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -275,13 +314,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -335,7 +368,15 @@ } ] }, - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of input files used in the expressions, statements, arguments and descriptions of the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -354,6 +395,10 @@ "$ref" : "#/definitions/textOutputChannel" }, "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -371,13 +416,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -431,7 +470,7 @@ } ] }, - "files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -449,12 +488,65 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" } } }, + "stdinData" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": ["content"], + "additionalProperties": false, + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + }, + { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + } + ] + }, "expressionOrStatement" : { "oneOf" : [ { @@ -488,9 +580,9 @@ } ] }, - "file" : { + "deprecatedFile": { "type" : "object", - "description" : "A file used in the test suite.", + "description" : "A file used in the test suite (DEPRECATED).", "required" : [ "name", "url" @@ -498,12 +590,30 @@ "properties" : { "name" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "url" : { "type" : "string", "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + "file" : { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "path" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." + }, + "content" : { + "type" : "string", + "description" : "The actual content of the file." } } }, @@ -515,10 +625,28 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "data" + "oneOf": [ + { + "required" : [ + "data" + ] + }, + { + "required" : [ + "content" + ] + } ], + "additionalProperties" : false, "properties" : { + "content": { + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "data" : { "$ref" : "#/definitions/textualType" }, @@ -530,17 +658,96 @@ } } }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : ["path"], + "additionalProperties" : false, + "properties" : { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "additionalProperties" : false, + "oneOf": [ + { + "required" : [ + "oracle", + "file", + "data" + ] + }, + { + "required" : [ + "oracle", + "file", + "content" + ] + } + ], + "properties" : { + "content": { + "$ref" : "#/definitions/textualType" + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + }, { "type" : "object", "description" : "Custom oracle for text values.", + "additionalProperties" : false, "required" : [ "oracle", "file", - "data" + "path" ], "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, "oracle" : { "const" : "custom_check" @@ -572,7 +779,7 @@ } ] }, - "fileOutputChannel": { + "deprecatedFileOutputChannel": { "anyOf" : [ { "type" : "object", @@ -646,6 +853,144 @@ } ] }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "array", + "description" : "Built-in oracle for files.", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "data" + ], + "properties" : { + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "data", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, "returnOutputChannel" : { "oneOf" : [ { @@ -865,7 +1210,8 @@ "string", "number", "integer", - "boolean" + "boolean", + "path" ] }, "yamlValue" : { @@ -873,7 +1219,8 @@ "not" : { "type" : [ "oracle", - "expression" + "expression", + "path" ] } }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f8d066bca..f9aa05648 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -39,7 +39,14 @@ } ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +104,15 @@ "tab" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +164,15 @@ "unit" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +252,15 @@ "testcases" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +282,15 @@ "script" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -275,13 +314,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -335,7 +368,15 @@ } ] }, - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -353,6 +394,14 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -367,13 +416,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -427,7 +470,15 @@ } ] }, - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -446,6 +497,10 @@ "$ref" : "#/definitions/textOutputChannel" }, "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -455,6 +510,51 @@ } } }, + "stdinData" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + } + ] + }, "expressionOrStatement" : { "oneOf" : [ { @@ -488,7 +588,7 @@ } ] }, - "file" : { + "deprecatedFile": { "type" : "object", "description" : "A file used in the test suite.", "required" : [ @@ -498,12 +598,30 @@ "properties" : { "name" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "url" : { "type" : "string", "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + "file" : { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "path" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." + }, + "content" : { + "type" : "string", + "description" : "The actual content of the file." } } }, @@ -515,13 +633,31 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "data" + "oneOf": [ + { + "required" : [ + "data" + ] + }, + { + "required" : [ + "content" + ] + } ], + "additionalProperties" : false, "properties" : { + "content": { + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." + }, "data" : { "$ref" : "#/definitions/textualType" }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "oracle" : { "const" : "builtin" }, @@ -530,17 +666,96 @@ } } }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : ["path"], + "additionalProperties" : false, + "properties" : { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "additionalProperties" : false, + "oneOf": [ + { + "required" : [ + "oracle", + "file", + "data" + ] + }, + { + "required" : [ + "oracle", + "file", + "content" + ] + } + ], + "properties" : { + "content": { + "$ref" : "#/definitions/textualType" + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + }, { "type" : "object", "description" : "Custom oracle for text values.", + "additionalProperties" : false, "required" : [ "oracle", "file", - "data" + "path" ], "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, "oracle" : { "const" : "custom_check" @@ -572,7 +787,7 @@ } ] }, - "fileOutputChannel": { + "deprecatedFileOutputChannel": { "anyOf" : [ { "type" : "object", @@ -646,6 +861,123 @@ } ] }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "array", + "description" : "Built-in oracle for files.", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "data" + ], + "properties" : { + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "data", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, "returnOutputChannel" : { "oneOf" : [ { diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 245b0cf7d..101b1c4c4 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,4 +1,5 @@ import json +import os import sys import textwrap from collections.abc import Callable @@ -31,7 +32,7 @@ StringTypes, resolve_to_basic, ) -from tested.dodona import ExtendedMessage +from tested.dodona import ExtendedMessage, Permission from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( @@ -53,23 +54,25 @@ ExitCodeOutputChannel, ExpectedException, FileOutputChannel, - FileUrl, GenericTextOracle, IgnoredChannel, + InputFile, LanguageLiterals, LanguageSpecificOracle, MainInput, Output, + OutputFileData, Suite, SupportedLanguage, Tab, Testcase, TextBuiltin, + TextChannelType, TextData, TextOutputChannel, ValueOutputChannel, ) -from tested.utils import get_args, recursive_dict_merge +from tested.utils import DataWithMessage, get_args, recursive_dict_merge YamlDict = dict[str, "YamlObject"] @@ -84,13 +87,26 @@ class ExpressionString(str): pass +class PathString(str): + pass + + class ReturnOracle(dict): pass OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | PathString ) @@ -130,6 +146,12 @@ def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString return ExpressionString(result) +def _path_string(loader: yaml.Loader, node: yaml.Node) -> PathString: + result = _parse_yaml_value(loader, node) + assert isinstance(result, str), f"A path must be a string, got {result}" + return PathString(result) + + def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: result = _parse_yaml_value(loader, node) assert isinstance( @@ -148,6 +170,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor("!path", _path_string, loader) try: return yaml.load(yaml_stream, loader) @@ -187,6 +210,10 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_path(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, PathString) + + def load_schema_validator( dsl_object: YamlObject = None, file: str = "schema-strict.json" ) -> Validator: @@ -215,9 +242,11 @@ def validate_tested_dsl_expression(value: object) -> bool: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression) + type_checker = ( + original_validator.TYPE_CHECKER.redefine("oracle", is_oracle) + .redefine("expression", is_expression) + .redefine("path", is_path) + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)( validate_tested_dsl_expression @@ -246,26 +275,46 @@ class DslContext: - The "files" property, which is a list of files. """ - files: list[FileUrl] = field(factory=list) + files: list[InputFile] = field(factory=list) config: dict[str, dict] = field(factory=dict) language: SupportedLanguage | Literal["tested"] = "tested" - def deepen_context(self, new_level: YamlDict | None) -> "DslContext": + def deepen_context( + self, new_level: YamlDict | None, workdir: Path | None + ) -> DataWithMessage["DslContext"]: """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. :param new_level: The new object from the DSL to get information from. + :param workdir: The working directory where all files are located. :return: A new context. """ if new_level is None: - return self + return DataWithMessage(data=self, messages=set()) + deprecated_usage = set() the_files = self.files - if "files" in new_level: - assert isinstance(new_level["files"], list) - additional_files = {_convert_file(f) for f in new_level["files"]} + if "input_files" in new_level or "files" in new_level: + key = "input_files" + if "files" in new_level: + key = "files" + deprecated_usage.add( + ExtendedMessage( + f"WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead.", + permission=Permission.STAFF, + ) + ) + + files = new_level[key] + assert isinstance(files, list) + additional_files = { + _convert_input_file( + f, workdir=workdir, not_deprecated_usage=len(deprecated_usage) == 0 + ) + for f in files + } the_files = list(set(self.files) | additional_files) the_config = self.config @@ -273,7 +322,10 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext": assert isinstance(new_level["config"], dict) the_config = recursive_dict_merge(the_config, new_level["config"]) - return evolve(self, files=the_files, config=the_config) + return DataWithMessage( + data=evolve(self, files=the_files, config=the_config), + messages=deprecated_usage, + ) def merge_inheritable_with_specific_config( self, level: YamlDict, config_name: str @@ -402,10 +454,26 @@ def _convert_value(value: YamlObject) -> Value: return _tested_type_to_value(tested_type) -def _convert_file(link_file: YamlDict) -> FileUrl: - assert isinstance(link_file["name"], str) - assert isinstance(link_file["url"], str) - return FileUrl(name=link_file["name"], url=link_file["url"]) +def _convert_input_file( + input_file: YamlDict, workdir: Path | None, not_deprecated_usage: bool +) -> InputFile: + path_key = "path" if not_deprecated_usage else "name" + path_str = input_file[path_key] + assert isinstance(path_str, str) + + content = "" + if "content" in input_file: + content = input_file["content"] + assert isinstance(content, str) + if workdir is not None: + full_path = workdir / path_str + dir_name = os.path.dirname(full_path) + if dir_name: + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + return InputFile(path=path_str, content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -454,42 +522,57 @@ def _convert_text_output_channel( ) -> TextOutputChannel: # Get the config applicable to this level. # Either attempt to get it from an object, or using the inherited options as is. + path = None + data = None if isinstance(stream, str): config = context.config.get(config_name, dict()) - raw_data = stream + data = stream else: assert isinstance(stream, dict) config = context.merge_inheritable_with_specific_config(stream, config_name) - raw_data = str(stream["data"]) + if "path" in stream: + path = str(stream["path"]) + + if "content" in stream or "data" in stream: + data = str(stream.get("content", stream.get("data"))) + + assert path or data # Normalize the data if necessary. - if config.get("normalizeTrailingNewlines", True): - data = _ensure_trailing_newline(raw_data) + if config.get("normalizeTrailingNewlines", True) and path is None: + assert data is not None + data = _ensure_trailing_newline(str(data)) + + if path is not None: + text_output = TextOutputChannel(data=data, path=path, type=TextChannelType.FILE) else: - data = raw_data + text_output = TextOutputChannel(data=data) if isinstance(stream, str): - return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) + text_output.oracle = GenericTextOracle(options=config) + return text_output else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": - return TextOutputChannel( - data=data, oracle=GenericTextOracle(options=config) - ) + + text_output.oracle = GenericTextOracle(options=config) + return text_output elif stream["oracle"] == "custom_check": - return TextOutputChannel( - data=data, oracle=_convert_custom_check_oracle(stream) - ) + text_output.oracle = _convert_custom_check_oracle(stream) + return text_output raise TypeError(f"Unknown text oracle type: {stream['oracle']}") -def _convert_file_output_channel( +def _convert_file_output_channel_deprecated( stream: YamlObject, context: DslContext, config_name: str ) -> FileOutputChannel: assert isinstance(stream, dict) - expected = str(stream["content"]) - actual = str(stream["location"]) + data = OutputFileData( + content_type=TextChannelType.TEXT, + content=str(stream["content"]), + path=str(stream["location"]), + ) if "oracle" not in stream or stream["oracle"] == "builtin": config = context.merge_inheritable_with_specific_config(stream, config_name) @@ -501,19 +584,70 @@ def _convert_file_output_channel( "line", ), f"The file oracle only supports modes full and line, not {config['mode']}" return FileOutputChannel( - expected_path=expected, - actual_path=actual, + output_data=[data], oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), ) elif stream["oracle"] == "custom_check": return FileOutputChannel( - expected_path=expected, - actual_path=actual, + output_data=[data], oracle=_convert_custom_check_oracle(stream), ) raise TypeError(f"Unknown file oracle type: {stream['oracle']}") +def _convert_output_file_channel( + output_files: YamlObject, context: DslContext, config_name: str +) -> FileOutputChannel: + + file_data = [] + data = output_files + if isinstance(output_files, dict): + data = output_files["data"] + assert isinstance(data, list) + + for item in data: + assert isinstance(item, dict) + content = item["content"] + if isinstance(content, PathString): + content_type = TextChannelType.FILE + else: + content = str(content) + content_type = TextChannelType.TEXT + + file_data.append( + OutputFileData( + content_type=content_type, + content=content, + path=str(item["path"]), + ) + ) + + if ( + not isinstance(output_files, dict) + or "oracle" not in output_files + or output_files["oracle"] == "builtin" + ): + level = {} if not isinstance(output_files, dict) else output_files + config = context.merge_inheritable_with_specific_config(level, config_name) + if "mode" not in config: + config["mode"] = "full" + + assert config["mode"] in ( + "full", + "line", + ), f"The file oracle only supports modes full and line, not {config['mode']}" + return FileOutputChannel( + output_data=file_data, + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) + elif output_files["oracle"] == "custom_check": + return FileOutputChannel( + output_data=file_data, + oracle=_convert_custom_check_oracle(output_files), + ) + raise TypeError(f"Unknown file oracle type: {output_files['oracle']}") + + def _convert_yaml_value(stream: YamlObject) -> Value | None: if isinstance(stream, ExpressionString): # We have an expression string. @@ -568,8 +702,12 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: - context = context.deepen_context(testcase) +def _convert_testcase( + testcase: YamlDict, context: DslContext, workdir: Path | None +) -> DataWithMessage[Testcase]: + context_with_data = context.deepen_context(testcase, workdir) + context = context_with_data.data + deprecated_messages = context_with_data.messages # This is backwards compatability to some extend. # TODO: remove this at some point. @@ -577,6 +715,7 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: testcase["expression"] = testcase.pop("statement") line_comment = "" + stdin_file = None _validate_testcase_combinations(testcase) if (expr_stmt := testcase.get("statement", testcase.get("expression"))) is not None: if isinstance(expr_stmt, dict) or context.language != "tested": @@ -599,8 +738,30 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - assert isinstance(testcase["stdin"], str) - stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) + stdin_data = testcase["stdin"] + data = None + path = "" + if isinstance(stdin_data, str): + data = _ensure_trailing_newline(stdin_data) + else: + assert isinstance(stdin_data, dict) + if "path" in stdin_data: + path = stdin_data["path"] + assert isinstance(path, str) + + if "content" in stdin_data: + content = stdin_data["content"] + assert isinstance(content, str) + data = _ensure_trailing_newline(content) + + if path: + stdin_file = InputFile( + path=path, + content=data if data is not None else "", + ) + stdin = TextData(data=data, path=path, type=TextChannelType.FILE) + else: + stdin = TextData(data=data) else: stdin = EmptyChannel.NONE arguments = testcase.get("arguments", []) @@ -616,7 +777,15 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout, context, "stdout") if (file := testcase.get("file")) is not None: - output.file = _convert_file_output_channel(file, context, "file") + output.file = _convert_file_output_channel_deprecated(file, context, "file") + deprecated_messages.add( + ExtendedMessage( + "WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead.", + permission=Permission.STAFF, + ) + ) + if (file := testcase.get("output_files")) is not None: + output.file = _convert_output_file_channel(file, context, "output_files") if (stderr := testcase.get("stderr")) is not None: output.stderr = _convert_text_output_channel(stderr, context, "stderr") if (exception := testcase.get("exception")) is not None: @@ -660,24 +829,43 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: else: the_description = None - return Testcase( - description=the_description, - input=the_input, - output=output, - link_files=context.files, - line_comment=line_comment, + input_files = context.files + if stdin_file is not None: + input_files = list(set(input_files) | {stdin_file}) + + return DataWithMessage( + data=Testcase( + description=the_description, + input=the_input, + output=output, + input_files=input_files, + line_comment=line_comment, + ), + messages=deprecated_messages, ) -def _convert_context(context: YamlDict, dsl_context: DslContext) -> Context: - dsl_context = dsl_context.deepen_context(context) +def _convert_context( + context: YamlDict, dsl_context: DslContext, workdir: Path | None +) -> DataWithMessage[Context]: + dsl_context_with_messages = dsl_context.deepen_context(context, workdir) + dsl_context = dsl_context_with_messages.data + deprecated_messages = dsl_context_with_messages.messages raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) - return Context(testcases=testcases) + testcases = _convert_dsl_list( + raw_testcases, dsl_context, workdir, _convert_testcase + ) + deprecated_messages.update(testcases.messages) + return DataWithMessage( + data=Context(testcases=testcases.data), + messages=deprecated_messages, + ) -def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: +def _convert_tab( + tab: YamlDict, context: DslContext, workdir: Path | None +) -> DataWithMessage[Tab]: """ Translate a DSL tab to a full test suite tab. @@ -685,51 +873,81 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: :param context: The context with config for the parent level. :return: A full tab. """ - context = context.deepen_context(tab) + context_with_messages = context.deepen_context(tab, workdir) + context = context_with_messages.data + deprecated_messages = context_with_messages.messages name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts_with_messages = _convert_dsl_list( + tab["contexts"], context, workdir, _convert_context + ) + contexts = contexts_with_messages.data + deprecated_messages.update(contexts_with_messages.messages) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts = _convert_dsl_list(tab["cases"], context, _convert_context) + contexts_with_messages = _convert_dsl_list( + tab["cases"], context, workdir, _convert_context + ) + contexts = contexts_with_messages.data + deprecated_messages.update(contexts_with_messages.messages) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases = _convert_dsl_list(tab["testcases"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in testcases] + testcases = _convert_dsl_list( + tab["testcases"], context, workdir, _convert_testcase + ) + deprecated_messages.update(testcases.messages) + contexts = [Context(testcases=[t]) for t in testcases.data] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases = _convert_dsl_list(tab["scripts"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in testcases] + testcases = _convert_dsl_list( + tab["scripts"], context, workdir, _convert_testcase + ) + deprecated_messages.update(testcases.messages) + contexts = [Context(testcases=[t]) for t in testcases.data] - return Tab(name=name, contexts=contexts) + return DataWithMessage( + data=Tab(name=name, contexts=contexts), + messages=deprecated_messages, + ) T = TypeVar("T") def _convert_dsl_list( - dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], T] -) -> list[T]: + dsl_list: list, + context: DslContext, + workdir: Path | None, + converter: Callable[[YamlDict, DslContext, Path | None], DataWithMessage[T]], +) -> DataWithMessage[list[T]]: """ Convert a list of YAML objects into a test suite object. """ objects = [] + deprecated_messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - objects.append(converter(dsl_object, context)) - return objects + ob = converter(dsl_object, context, workdir) + deprecated_messages.update(ob.messages) + objects.append(ob.data) + return DataWithMessage( + data=objects, + messages=deprecated_messages, + ) -def _convert_dsl(dsl_object: YamlObject) -> Suite: +def _convert_dsl( + dsl_object: YamlObject, workdir: Path | None +) -> DataWithMessage[Suite]: """ Translate a DSL test suite into a full test suite. @@ -740,38 +958,51 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: :return: A full test suite. """ context = DslContext() + deprecated_messages = set() + if isinstance(dsl_object, list): namespace = None tab_list = dsl_object else: assert isinstance(dsl_object, dict) namespace = dsl_object.get("namespace") - context = context.deepen_context(dsl_object) + context_with_messages = context.deepen_context(dsl_object, workdir) + context = context_with_messages.data + deprecated_messages.update(context_with_messages.messages) tab_list = dsl_object.get("units", dsl_object.get("tabs")) assert isinstance(tab_list, list) if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs = _convert_dsl_list(tab_list, context, _convert_tab) + tabs = _convert_dsl_list(tab_list, context, workdir, _convert_tab) + deprecated_messages.update(tabs.messages) if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace) + return DataWithMessage( + data=Suite(tabs=tabs.data, namespace=namespace), + messages=deprecated_messages, + ) else: - return Suite(tabs=tabs) + return DataWithMessage( + data=Suite(tabs=tabs.data), + messages=deprecated_messages, + ) -def parse_dsl(dsl_string: str) -> Suite: +def parse_dsl(dsl_string: str, workdir: Path | None = None) -> DataWithMessage[Suite]: """ Parse a string containing a DSL test suite into our representation, a test suite. :param dsl_string: The string containing a DSL. + :param workdir: The working directory for the test suite. :return: The parsed and converted test suite. """ dsl_object = _parse_yaml(dsl_string) _validate_dsl(dsl_object) - return _convert_dsl(dsl_object) + + return _convert_dsl(dsl_object, workdir) def translate_to_test_suite(dsl_string: str) -> str: @@ -779,7 +1010,8 @@ def translate_to_test_suite(dsl_string: str) -> str: Convert a DSL to a test suite. :param dsl_string: The DSL. + :param workdir: The working directory for the test suite. :return: The test suite. """ - suite = parse_dsl(dsl_string) - return suite_to_json(suite) + suite = parse_dsl(dsl_string, Path(".")) + return suite_to_json(suite.data) diff --git a/tested/judge/core.py b/tested/judge/core.py index 24bdd3b31..bddaaeb0e 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -116,6 +116,9 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) + if bundle.messages: + collector.add_messages(bundle.messages) + max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() @@ -338,27 +341,35 @@ def _process_results( compilation_results=compilation_results, ) + input_files = [] + for case in planned.context.testcases: + for file in case.input_files: + file_data = {"path": file.path} + if file.content != "": + file_data["content"] = file.content + input_files.append(file_data) + if not input_files: + input_files = None + if bundle.language.supports_debug_information(): - # TODO: this is currently very Python-specific - # See if we need a callback to the language modules in the future. - # TODO: we could probably re-use the "readable_input" function here, - # since it only differs a bit. meta_statements = [] meta_stdin = None + for case in planned.context.testcases: if case.is_main_testcase(): assert isinstance(case.input, MainInput) - if isinstance(case.input.stdin, TextData): - meta_stdin = case.input.stdin.get_data_as_string( - bundle.config.resources - ) + if ( + isinstance(case.input.stdin, TextData) + and case.input.stdin.data is not None + ): + meta_stdin = case.input.stdin.data elif isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) elif isinstance(case.input, LanguageLiterals): stmt = case.input.get_for(bundle.config.programming_language) meta_statements.append(stmt) - else: + elif not case.is_main_testcase(): raise AssertionError(f"Found unknown case input type: {case.input}") if meta_statements: @@ -367,17 +378,24 @@ def _process_results( # Don't add empty statements meta_statements = None + if not input_files: + input_files = None + collector.add( CloseContext( data=Metadata( - statements=meta_statements, - stdin=meta_stdin, + statements=meta_statements, files=input_files, stdin=meta_stdin ) ), planned.context_index, ) else: - collector.add(CloseContext(), planned.context_index) + collector.add( + CloseContext( + data=Metadata(files=input_files, stdin=None, statements=None) + ), + planned.context_index, + ) if continue_ in (Status.TIME_LIMIT_EXCEEDED, Status.MEMORY_LIMIT_EXCEEDED): return continue_, currently_open_tab diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 44b2571ab..f64aa8624 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -1,6 +1,4 @@ -import html import logging -from collections.abc import Collection from enum import StrEnum, unique from pathlib import Path from typing import Literal @@ -42,9 +40,9 @@ ExitCodeOutputChannel, FileOutput, FileOutputChannel, - FileUrl, IgnoredChannel, OutputChannel, + OutputFileData, SpecialOutputChannel, Testcase, TextOutput, @@ -109,40 +107,54 @@ def _evaluate_channel( evaluator = get_oracle( bundle, context_directory, output, testcase, unexpected_status=unexpected_status ) - # Run the oracle. - evaluation_result = evaluator(output, actual if actual else "") - status = evaluation_result.result - - # Decide if we should show this channel or not. - is_correct = status.enum == Status.CORRECT - should_report_case = should_show(output, channel, evaluation_result) - - if not should_report_case and is_correct: - # We do report that a test is correct, to set the status. - return False - - expected = evaluation_result.readable_expected - out.add(StartTest(expected=expected, channel=channel)) - - # Report any messages we received. - for message in evaluation_result.messages: - out.add(AppendMessage(message=message)) - - missing = False - if actual is None: - out.add(AppendMessage(message=get_i18n_string("judge.evaluation.missing"))) - missing = True - elif should_report_case and timeout and not is_correct: - status.human = get_i18n_string("judge.evaluation.time-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - elif should_report_case and memory and not is_correct: - status.human = get_i18n_string("judge.evaluation.memory-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - - # Close the test. - out.add(CloseTest(generated=evaluation_result.readable_actual, status=status)) + + if channel == Channel.FILE and output != "ignored": + assert isinstance(output, FileOutputChannel) + output_channels = output.output_data + else: + output_channels = [output] + + missing = actual is None + + for output_element in output_channels: + # Run the oracle. + evaluation_result = evaluator(output_element, actual if actual else "") + status = evaluation_result.result + + # Decide if we should show this channel or not. + is_correct = status.enum == Status.CORRECT + should_report_case = should_show(output, channel, evaluation_result) + + if should_report_case or not is_correct: + expected = evaluation_result.readable_expected + expected_channel = channel + if isinstance(output_element, OutputFileData): + expected_channel = f"file: {output_element.path}" + out.add(StartTest(expected=expected, channel=expected_channel)) + + # Report any messages we received. + for message in evaluation_result.messages: + out.add(AppendMessage(message=message)) + + if actual is None: + out.add( + AppendMessage(message=get_i18n_string("judge.evaluation.missing")) + ) + elif should_report_case and timeout and not is_correct: + status.human = get_i18n_string("judge.evaluation.time-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + elif should_report_case and memory and not is_correct: + status.human = get_i18n_string("judge.evaluation.memory-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + + # Close the test. + out.add( + CloseTest(generated=evaluation_result.readable_actual, status=status) + ) + else: + missing = False return missing @@ -241,15 +253,11 @@ def evaluate_context_results( ) ) - # All files that will be used in this context. - all_files = context.get_files() - # Begin processing the normal testcases. for i, testcase in enumerate(context.testcases): _logger.debug(f"Evaluating testcase {i}") - readable_input, seen = get_readable_input(bundle, testcase) - all_files = all_files - seen + readable_input = get_readable_input(bundle, testcase) t_col = TestcaseCollector(StartTestcase(description=readable_input)) # Get the functions @@ -351,10 +359,6 @@ def evaluate_context_results( t_col.to_manager(collector, CloseTestcase(), i) - # Add file links - if all_files: - collector.add(_link_files_message(all_files)) - if exec_results.timeout: return Status.TIME_LIMIT_EXCEEDED if exec_results.memory: @@ -362,20 +366,6 @@ def evaluate_context_results( return None -def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: - link_list = ", ".join( - f'' - f'{html.escape(link_file.name)}' - for link_file in link_files - ) - file_list_str = get_i18n_string( - "judge.evaluation.files", count=len(link_files), files=link_list - ) - description = f"

{file_list_str}

" - message = ExtendedMessage(description=description, format="html") - return AppendMessage(message=message) - - def should_show( test: OutputChannel, channel: Channel, result: OracleResult | None = None ) -> bool: @@ -495,15 +485,12 @@ def complete_evaluation(bundle: Bundle, collector: OutputManager): ] if testcase_start == 0: collector.add(StartContext(description=context.description)) - # All files that will be used in this context. - all_files = context.get_files() # Begin normal testcases. for j, testcase in enumerate( context.testcases[testcase_start:], start=testcase_start ): - readable_input, seen = get_readable_input(bundle, testcase) - all_files = all_files - seen + readable_input = get_readable_input(bundle, testcase) updates.append(StartTestcase(description=readable_input)) # Do the normal output channels. @@ -521,10 +508,6 @@ def complete_evaluation(bundle: Bundle, collector: OutputManager): updates.append(CloseTestcase(accepted=False)) testcase_start = 0 # For the next context, start at the beginning - # Add links to files we haven't seen yet. - if all_files: - updates.insert(0, _link_files_message(all_files)) - collector.add_all(updates) collector.add(CloseContext(accepted=False)) collector.add(CloseTab()) diff --git a/tested/judge/execution.py b/tested/judge/execution.py index c5ee6f42c..bc1e903e5 100644 --- a/tested/judge/execution.py +++ b/tested/judge/execution.py @@ -241,7 +241,7 @@ def execute_unit( executable = executable_or_status files.remove(executable) - stdin = unit.get_stdin(bundle.config.resources) + stdin = unit.get_stdin(bundle.config.workdir) # Do the execution. base_result = execute_file( diff --git a/tested/languages/generation.py b/tested/languages/generation.py index 7c7d883b4..ebf5f7825 100644 --- a/tested/languages/generation.py +++ b/tested/languages/generation.py @@ -2,15 +2,9 @@ Translates items from the test suite into the actual programming language. """ -import html -import json import logging -import re import shlex -import urllib.parse -from collections.abc import Iterable from pathlib import Path -from re import Match from typing import TYPE_CHECKING, TypeAlias from pygments import highlight @@ -29,16 +23,8 @@ prepare_execution_unit, prepare_expression, ) -from tested.parsing import get_converter from tested.serialisation import Expression, Statement, VariableType -from tested.testsuite import ( - Context, - FileUrl, - LanguageLiterals, - MainInput, - Testcase, - TextData, -) +from tested.testsuite import Context, LanguageLiterals, MainInput, Testcase, TextData from tested.utils import is_statement_strict if TYPE_CHECKING: @@ -77,20 +63,6 @@ def generate_execution_unit( return bundle.language.generate_execution_unit(prepared_execution) -def _handle_link_files(link_files: Iterable[FileUrl], language: str) -> tuple[str, str]: - dict_links = dict( - (link_file.name, get_converter().unstructure(link_file)) - for link_file in link_files - ) - files = json.dumps(dict_links) - return ( - f"
",
-        "
", - ) - - def _get_heredoc_token(stdin: str) -> str: delimiter = "STDIN" while delimiter in stdin: @@ -98,9 +70,7 @@ def _get_heredoc_token(stdin: str) -> str: return delimiter -def get_readable_input( - bundle: Bundle, case: Testcase -) -> tuple[ExtendedMessage, set[FileUrl]]: +def get_readable_input(bundle: Bundle, case: Testcase) -> ExtendedMessage: """ Get human-readable input for a testcase. This function will use, in order of availability: @@ -108,7 +78,7 @@ def get_readable_input( 1. A description on the testcase. 2. If it is a normal testcase: a. A function expression or generate_statement. - 3. If it is a context testcase: + 3. If it is a context testcase (main-testcase): a. The stdin and the arguments. """ format_ = "text" # By default, we use text as input. @@ -127,21 +97,32 @@ def get_readable_input( command = shlex.join([submission] + case.input.arguments) args = f"$ {command}" # Determine the stdin - if isinstance(case.input.stdin, TextData): - stdin = case.input.stdin.get_data_as_string(bundle.config.resources) - else: - stdin = "" - - # If we have both stdin and arguments, we use a here-document. - if case.input.arguments and stdin: - assert stdin[-1] == "\n", "stdin must end with a newline" - delimiter = _get_heredoc_token(stdin) - text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" - elif stdin: - assert not case.input.arguments - text = stdin + stdin_data = case.input.stdin + stdin = "" + if isinstance(stdin_data, TextData): + if stdin_data.type == "file": + stdin = stdin_data.path + else: + stdin = stdin_data.data + assert stdin is not None + + # If we have both stdin and arguments, we use a here-document or here-string. + if stdin: + if isinstance(stdin_data, TextData) and stdin_data.type == "file": + text = f"{args} < {stdin}" + elif case.input.arguments: + assert stdin[-1] == "\n", "stdin must end with a newline" + if stdin.count("\n") > 1: + delimiter = _get_heredoc_token(stdin) + text = f"{args} << '{delimiter}'\n{stdin}{delimiter}" + else: + text = f"{args} <<< {stdin.strip()}" + else: + assert not case.input.arguments + text = stdin else: text = args + elif isinstance(case.input, Statement): format_ = bundle.config.programming_language text = generate_statement(bundle, case.input) @@ -157,59 +138,14 @@ def get_readable_input( if case.line_comment: text = f"{text} {bundle.language.comment(case.line_comment)}" - # If there are no files, return now. This means we don't need to do ugly stuff. - if not case.link_files: - return ExtendedMessage(description=text, format=format_), set() - - # We have potential files. - # Check if the file names are present in the string. - # If not, we can also stop before doing ugly things. - # We construct a regex, since that can be faster than checking everything. - simple_regex = re.compile( - "|".join(map(lambda x: re.escape(x.name), case.link_files)) - ) - - if not simple_regex.search(text): - # There is no match, so bail now. - return ExtendedMessage(description=text, format=format_), set() - - # Now we need to do ugly stuff. - # Begin by compiling the HTML that will be displayed. - if format_ == "text": - generated_html = html.escape(text) - elif format_ == "console": - generated_html = highlight_code(text) - else: - generated_html = highlight_code(text, bundle.config.programming_language) - - # Map of file URLs. - url_map = {html.escape(x.name): x for x in case.link_files} - - seen = set() - escaped_regex = re.compile("|".join(url_map.keys())) - - # Replaces the match with the corresponding link. - def replace_link(match: Match) -> str: - filename = match.group() - the_file = url_map[filename] - the_url = urllib.parse.quote(the_file.url) - the_replacement = ( - f'{filename}' - ) - seen.add(the_file) - return the_replacement - - generated_html = escaped_regex.sub(replace_link, generated_html) - prefix, suffix = _handle_link_files(seen, format_) - generated_html = f"{prefix}{generated_html}{suffix}" - return ExtendedMessage(description=generated_html, format="html"), seen + return ExtendedMessage(description=text, format=format_) def attempt_readable_input(bundle: Bundle, context: Context) -> ExtendedMessage: # Try until we find a testcase with input. testcases = context.testcases for testcase in testcases: - result, _ = get_readable_input(bundle, testcase) + result = get_readable_input(bundle, testcase) if result.description: return result diff --git a/tested/main.py b/tested/main.py index b45490282..45fc4ef28 100644 --- a/tested/main.py +++ b/tested/main.py @@ -29,11 +29,14 @@ def run(config: DodonaConfig, judge_output: IO): _, ext = os.path.splitext(config.test_suite) is_yaml = ext.lower() in (".yaml", ".yml") + messages = set() if is_yaml: - suite = parse_dsl(textual_suite) + suite_with_message = parse_dsl(textual_suite, config.workdir) + messages = suite_with_message.messages + suite = suite_with_message.data else: suite = parse_test_suite(textual_suite) - pack = create_bundle(config, judge_output, suite) + pack = create_bundle(config, judge_output, suite, messages=messages) from .judge import judge judge(pack) diff --git a/tested/oracles/text.py b/tested/oracles/text.py index 6724e18b9..ecc1564f6 100644 --- a/tested/oracles/text.py +++ b/tested/oracles/text.py @@ -8,7 +8,12 @@ from tested.dodona import Status, StatusMessage from tested.internationalization import get_i18n_string from tested.oracles.common import OracleConfig, OracleResult -from tested.testsuite import FileOutputChannel, OutputChannel, TextOutputChannel +from tested.testsuite import ( + OutputChannel, + OutputFileData, + TextChannelType, + TextOutputChannel, +) def _is_number(string: str) -> float | None: @@ -118,7 +123,7 @@ def evaluate_file( When no mode is passed, the oracle will default to ``full``. """ - assert isinstance(channel, FileOutputChannel) + assert isinstance(channel, OutputFileData) options = _text_options(config) # There must be nothing as output. @@ -134,15 +139,17 @@ def evaluate_file( messages=[message], ) - expected_path = f"{config.bundle.config.resources}/{channel.expected_path}" + expected = channel.content + if channel.content_type == TextChannelType.FILE: + expected_path = f"{config.bundle.config.resources}/{expected}" - try: - with open(expected_path, "r") as file: - expected = file.read() - except FileNotFoundError: - raise ValueError(f"File {expected_path} not found in resources.") + try: + with open(expected_path, "r") as file: + expected = file.read() + except FileNotFoundError: + raise ValueError(f"File {expected_path} not found in resources.") - actual_path = config.context_dir / channel.actual_path + actual_path = config.context_dir / channel.path try: with open(str(actual_path), "r") as file: diff --git a/tested/testsuite.py b/tested/testsuite.py index 4ef5f0aac..1f086bcce 100644 --- a/tested/testsuite.py +++ b/tested/testsuite.py @@ -247,15 +247,19 @@ def _resolve_path(working_directory, file_path): class TextData(WithFeatures): """Describes textual data: either directly or in a file.""" - data: str + data: str | None + path: str = "" type: TextChannelType = TextChannelType.TEXT def get_data_as_string(self, working_directory: Path) -> str: """Get the data as a string, reading the file if necessary.""" - if self.type == TextChannelType.TEXT: + if self.data is not None: return self.data + + if self.type == TextChannelType.TEXT: + return "" elif self.type == TextChannelType.FILE: - file_path = _resolve_path(working_directory, self.data) + file_path = _resolve_path(working_directory, self.path) with open(file_path, "r") as file: return file.read() else: @@ -274,14 +278,23 @@ class TextOutputChannel(TextData): oracle: GenericTextOracle | CustomCheckOracle = field(factory=GenericTextOracle) +@define(frozen=True) +class OutputFileData: + content_type: TextChannelType + content: str + path: str + oracle: GenericTextOracle | CustomCheckOracle = field( + factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) + ) + + @fallback_field(get_converter(), {"evaluator": "oracle"}) @ignore_field(get_converter(), "show_expected") @define class FileOutputChannel(WithFeatures): """Describes the output for files.""" - expected_path: str # Path to the file to compare to. - actual_path: str # Path to the generated file (by the user code) + output_data: list[OutputFileData] oracle: GenericTextOracle | CustomCheckOracle = field( factory=lambda: GenericTextOracle(name=TextBuiltin.FILE) ) @@ -290,9 +303,16 @@ def get_used_features(self) -> FeatureSet: return NOTHING def get_data_as_string(self, resources: Path) -> str: - file_path = _resolve_path(resources, self.expected_path) - with open(file_path, "r") as file: - return file.read() + file_content = [] + for i in range(len(self.output_data)): + output_data = self.output_data[i] + if output_data.content_type == TextChannelType.FILE: + file_path = _resolve_path(resources, output_data.content) + with open(file_path, "r") as file: + file_content.append(f"{file.read()}") + else: + file_content.append(f"{output_data.content[i]}") + return "\n".join(file_content) @fallback_field(get_converter(), {"evaluator": "oracle"}) @@ -412,7 +432,11 @@ def get_used_features(self) -> FeatureSet: SpecialOutputChannel = EmptyChannel | IgnoredChannel OracleOutputChannel = Union[ - TextOutputChannel, FileOutputChannel, ValueOutputChannel, ExceptionOutputChannel + TextOutputChannel, + FileOutputChannel, + OutputFileData, + ValueOutputChannel, + ExceptionOutputChannel, ] NormalOutputChannel = OracleOutputChannel | ExitCodeOutputChannel @@ -528,9 +552,9 @@ def get_functions(self) -> Iterable[FunctionCall]: @define(frozen=True) -class FileUrl: - url: str - name: str +class InputFile: + path: str + content: str = "" @ignore_field(get_converter(), "essential") @@ -557,7 +581,7 @@ class Testcase(WithFeatures, WithFunctions): input: Statement | MainInput | LanguageLiterals description: Message | None = None output: Output = field(factory=Output) - link_files: list[FileUrl] = field(factory=list) + input_files: list[InputFile] = field(factory=list) line_comment: str = "" def get_used_features(self) -> FeatureSet: @@ -665,10 +689,10 @@ def has_main_testcase(self): def has_exit_testcase(self): return not self.testcases[-1].output.exit_code == IgnoredChannel.IGNORED - def get_files(self) -> set[FileUrl]: + def get_files(self) -> set[InputFile]: all_files = set() for t in self.testcases: - all_files = all_files.union(t.link_files) + all_files = all_files.union(t.input_files) return all_files diff --git a/tested/utils.py b/tested/utils.py index e913c8b6f..6b5c838e1 100644 --- a/tested/utils.py +++ b/tested/utils.py @@ -7,9 +7,13 @@ from collections.abc import Callable, Iterable from itertools import zip_longest from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, TypeGuard, TypeVar +from typing import IO, TYPE_CHECKING, Any, Generic, TypeGuard, TypeVar from typing import get_args as typing_get_args +from attr import define + +from tested.dodona import ExtendedMessage + if TYPE_CHECKING: from tested.serialisation import Assignment @@ -56,6 +60,12 @@ def get_identifier() -> str: T = TypeVar("T") +@define +class DataWithMessage(Generic[T]): + data: T + messages: set[ExtendedMessage] + + def get_args(type_: Any) -> tuple[Any, ...]: """ Get the args of a type or the type itself. diff --git a/tests/exercises/echo-function-file-output/evaluation/one.yaml b/tests/exercises/echo-function-file-output/evaluation/one.yaml index a428bf276..32775331a 100644 --- a/tests/exercises/echo-function-file-output/evaluation/one.yaml +++ b/tests/exercises/echo-function-file-output/evaluation/one.yaml @@ -1,7 +1,8 @@ - tab: "Test" testcases: - statement: echo_function("result.txt", "Hallo") - file: - content: "contents.txt" - location: "result.txt" + output_files: + data: + - content: !path "contents.txt" + path: "result.txt" oracle: builtin diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py index aecf78846..fe335b66a 100644 --- a/tests/test_dsl_yaml.py +++ b/tests/test_dsl_yaml.py @@ -1,5 +1,6 @@ # type: ignore[reportAttributeAccessIssue] import json +import os from pathlib import Path import pytest @@ -19,6 +20,7 @@ SequenceTypes, StringTypes, ) +from tested.dodona import Permission from tested.dsl import parse_dsl, translate_to_test_suite from tested.dsl.translate_parser import load_schema_validator from tested.serialisation import ( @@ -32,12 +34,13 @@ from tested.testsuite import ( CustomCheckOracle, FileOutputChannel, - FileUrl, GenericTextOracle, GenericValueOracle, + InputFile, LanguageLiterals, LanguageSpecificOracle, SupportedLanguage, + TextChannelType, TextOutputChannel, ValueOutputChannel, parse_test_suite, @@ -69,9 +72,49 @@ def test_parse_one_tab_ctx(): tc = context.testcases[0] assert tc.is_main_testcase() assert tc.input.stdin.data == "Input string\n" + assert tc.input.stdin.type == TextChannelType.TEXT assert tc.input.arguments == ["--arg", "argument"] assert tc.output.stderr.data == "Error string\n" + assert tc.output.stderr.type == TextChannelType.TEXT assert tc.output.stdout.data == "Output string\n" + assert tc.output.stdout.type == TextChannelType.TEXT + assert tc.output.exit_code.value == 1 + + +def test_parse_one_tab_ctx_with_files(): + yaml_str = """ +namespace: "solution" +tabs: +- tab: "Ctx" + testcases: + - arguments: [ "--arg", "argument" ] + stdin: + path: "input.text" + stdout: + path: "output.text" + stderr: + path: "error.text" + exit_code: 1 + """ + json_str = translate_to_test_suite(yaml_str) + suite = parse_test_suite(json_str) + assert suite.namespace == "solution" + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert tab.name == "Ctx" + assert len(tab.contexts) == 1 + context = tab.contexts[0] + assert len(context.testcases) == 1 + tc = context.testcases[0] + assert tc.is_main_testcase() + assert tc.input.stdin.path == "input.text" + assert tc.input.stdin.type == TextChannelType.FILE + assert tc.input.stdin.data is None + assert tc.input.arguments == ["--arg", "argument"] + assert tc.output.stderr.path == "error.text" + assert tc.output.stderr.type == TextChannelType.FILE + assert tc.output.stdout.path == "output.text" + assert tc.output.stdout.type == TextChannelType.FILE assert tc.output.exit_code.value == 1 @@ -532,7 +575,8 @@ def test_tab_config_trickles_down_stderr(): namespace: "solution" testcases: - arguments: [ "--arg", "argument" ] - stdin: "Input string" + stdin: + content: "Input string" stdout: "Output string" stderr: "Error string" exit_code: 1 @@ -744,16 +788,84 @@ def test_value_built_in_checks_implied(): ) -def test_file_custom_check_correct(): +def test_using_deprecated_file(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - statement: 'test()' + file: + content: "Hello world!" + location: "test.txt" + """ + suite_with_data = parse_dsl(yaml_str) + suite = suite_with_data.data + messages = list(suite_with_data.messages) + assert len(messages) == 1 + deprecated_message = messages[0] + assert deprecated_message.permission == Permission.STAFF + assert ( + deprecated_message.description + == "WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead." + ) + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert len(tab.contexts) == 1 + testcases = tab.contexts[0].testcases + assert len(testcases) == 1 + test = testcases[0] + assert isinstance(test.input, FunctionCall) + assert isinstance(test.output.file, FileOutputChannel) + assert test.output.file.output_data[0].path == "test.txt" + assert test.output.file.output_data[0].content == "Hello world!" + assert test.output.file.output_data[0].content_type == TextChannelType.TEXT + + +def test_using_deprecated_files(): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test("hello.txt")' + return: "Hello world!" + files: + - url: "media/hello.txt" + name: "hello.txt" + """ + suite_with_data = parse_dsl(yaml_str) + suite = suite_with_data.data + messages = list(suite_with_data.messages) + assert len(messages) == 1 + deprecated_message = messages[0] + assert deprecated_message.permission == Permission.STAFF + assert ( + deprecated_message.description + == "WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead." + ) + assert len(suite.tabs) == 1 + tab = suite.tabs[0] + assert len(tab.contexts) == 1 + testcases = tab.contexts[0].testcases + assert len(testcases) == 1 + test = testcases[0] + assert isinstance(test.input, FunctionCall) + assert len(test.input_files) == 1 + assert test.input_files[0].path == "hello.txt" + + +def test_output_files_custom_check_correct(): yaml_str = f""" - tab: 'Test' contexts: - testcases: - statement: 'test()' - file: - content: "test/hallo.txt" + output_files: + data: + - content: !path "test/hallo.txt" + path: "test.txt" + - content: "Hallo world!" + path: "test2.txt" oracle: "custom_check" - location: "test.txt" name: "evaluate_test" file: "test.py" """ @@ -768,8 +880,12 @@ def test_file_custom_check_correct(): assert isinstance(test.input, FunctionCall) assert isinstance(test.output.file, FileOutputChannel) assert isinstance(test.output.file.oracle, CustomCheckOracle) - assert test.output.file.actual_path == "test.txt" - assert test.output.file.expected_path == "test/hallo.txt" + assert test.output.file.output_data[0].path == "test.txt" + assert test.output.file.output_data[0].content == "test/hallo.txt" + assert test.output.file.output_data[0].content_type == TextChannelType.FILE + assert test.output.file.output_data[1].path == "test2.txt" + assert test.output.file.output_data[1].content == "Hallo world!" + assert test.output.file.output_data[1].content_type == TextChannelType.TEXT oracle = test.output.file.oracle assert oracle.function.name == "evaluate_test" assert oracle.function.file == Path("test.py") @@ -1172,19 +1288,16 @@ def test_additional_properties_are_not_allowed(): def test_files_are_propagated(): yaml_str = """ - tab: "Config ctx" - files: - - name: "test" - url: "test.md" - - name: "two" - url: "two.md" + input_files: + - path: "test" + - path: "two" testcases: - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.34" - arguments: [ '-a', '2.125', '1.212' ] stdout: "3.337" - files: - - name: "test" - url: "twooo.md" + input_files: + - path: "test" """ json_str = translate_to_test_suite(yaml_str) suite = parse_test_suite(json_str) @@ -1192,12 +1305,31 @@ def test_files_are_propagated(): ctx0, ctx1 = tab.contexts testcases0, testcases1 = ctx0.testcases, ctx1.testcases test0, test1 = testcases0[0], testcases1[0] - assert set(test0.link_files) == { - FileUrl(name="test", url="test.md"), - FileUrl(name="two", url="two.md"), + assert set(test0.input_files) == { + InputFile(path="test"), + InputFile(path="two"), } +def test_input_file_created(tmp_path: Path, pytestconfig: pytest.Config): + yaml_str = f""" + - tab: 'Test' + contexts: + - testcases: + - expression: 'test("hello.txt")' + return: "Hello world!" + input_files: + - content: "Hello world!" + path: "hello.txt" + """ + + os.chdir(tmp_path) + translate_to_test_suite(yaml_str) + + with open("hello.txt", "r", encoding="utf-8") as f: + assert f.read() == "Hello world!" + + def test_newlines_are_added_to_stdout(): yaml_str = """ - unit: "Statement and main" diff --git a/tests/test_functionality.py b/tests/test_functionality.py index da5aeb804..3b692c451 100644 --- a/tests/test_functionality.py +++ b/tests/test_functionality.py @@ -18,7 +18,15 @@ from tested.judge.execution import ExecutionResult from tested.languages import LANGUAGES, get_language from tested.languages.generation import get_readable_input -from tested.testsuite import Context, MainInput, Suite, Tab, Testcase, TextData +from tested.testsuite import ( + Context, + MainInput, + Suite, + Tab, + Testcase, + TextChannelType, + TextData, +) from tests.language_markers import ( ALL_LANGUAGES, ALL_SPECIFIC_LANGUAGES, @@ -765,7 +773,7 @@ def test_main_call_quotes(tmp_path: Path, pytestconfig: pytest.Config): ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) bundle = create_bundle(conf, sys.stdout, suite) - actual, _ = get_readable_input(bundle, the_input) + actual = get_readable_input(bundle, the_input) assert ( actual.description == "$ submission hello 'it'\"'\"'s' '$yes' --hello=no -hello" @@ -788,7 +796,7 @@ def test_stdin_and_arguments_use_heredoc(tmp_path: Path, pytestconfig: pytest.Co ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) bundle = create_bundle(conf, sys.stdout, suite) - actual, _ = get_readable_input(bundle, the_input) + actual = get_readable_input(bundle, the_input) assert ( actual.description @@ -810,8 +818,53 @@ def test_stdin_token_is_unique(tmp_path: Path, pytestconfig: pytest.Config): ) suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) bundle = create_bundle(conf, sys.stdout, suite) - actual, _ = get_readable_input(bundle, the_input) + actual = get_readable_input(bundle, the_input) assert ( actual.description == "$ submission hello << 'STDINN'\nOne line\nSTDIN\nSTDINN" ) + + +def test_stdin_with_path(tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, + "echo-function", + "bash", + tmp_path, + "two.yaml", + "top-level-output", + ) + the_input = Testcase( + input=MainInput( + arguments=["hello"], + stdin=TextData( + data="One line\n", + path="line.txt", + type=TextChannelType.FILE, + ), + ) + ) + suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) + bundle = create_bundle(conf, sys.stdout, suite) + actual = get_readable_input(bundle, the_input) + + assert actual.description == "$ submission hello < line.txt" + + +def test_stdin_with_one_line(tmp_path: Path, pytestconfig: pytest.Config): + conf = configuration( + pytestconfig, + "echo-function", + "bash", + tmp_path, + "two.yaml", + "top-level-output", + ) + the_input = Testcase( + input=MainInput(arguments=["hello"], stdin=TextData(data="One line\n")) + ) + suite = Suite(tabs=[Tab(contexts=[Context(testcases=[the_input])], name="hallo")]) + bundle = create_bundle(conf, sys.stdout, suite) + actual = get_readable_input(bundle, the_input) + + assert actual.description == "$ submission hello <<< One line" diff --git a/tests/test_oracles_builtin.py b/tests/test_oracles_builtin.py index 7e21ffa02..3898969d4 100644 --- a/tests/test_oracles_builtin.py +++ b/tests/test_oracles_builtin.py @@ -24,9 +24,10 @@ from tested.testsuite import ( ExceptionOutputChannel, ExpectedException, - FileOutputChannel, + OutputFileData, Suite, SupportedLanguage, + TextChannelType, TextOutputChannel, ValueOutputChannel, ) @@ -148,8 +149,10 @@ def test_file_oracle_full_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "actual\nactual") @@ -170,8 +173,36 @@ def test_file_oracle_full_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + result = evaluate_file(config, channel, "") + s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") + assert result.result.enum == Status.CORRECT + assert result.readable_expected == "expected\nexpected" + assert result.readable_actual == "expected\nexpected" + + +def test_file_oracle_full_correct_with_text_content( + tmp_path: Path, pytestconfig: pytest.Config, mocker: MockerFixture +): + config = oracle_config(tmp_path, pytestconfig, {"mode": "full"}) + s = mocker.spy( + tested.oracles.text, name="compare_text" # type: ignore[reportAttributeAccessIssue] + ) + mock_files = [ + mocker.mock_open(read_data=content).return_value + for content in ["expected\nexpected", "expected\nexpected"] + ] + mock_opener = mocker.mock_open() + mock_opener.side_effect = mock_files + mocker.patch("builtins.open", mock_opener) + channel = OutputFileData( + content="expected\nexpected", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_called_once_with(ANY, "expected\nexpected", "expected\nexpected") @@ -194,8 +225,10 @@ def test_file_oracle_line_wrong( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "actual") @@ -220,8 +253,10 @@ def test_file_oracle_line_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") @@ -246,8 +281,10 @@ def test_file_oracle_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected", "expected") @@ -272,8 +309,10 @@ def test_file_oracle_dont_strip_lines_correct( mock_opener = mocker.mock_open() mock_opener.side_effect = mock_files mocker.patch("builtins.open", mock_opener) - channel = FileOutputChannel( - expected_path="expected.txt", actual_path="expected.txt" + channel = OutputFileData( + content="expected.txt", + path="expected.txt", + content_type=TextChannelType.FILE, ) result = evaluate_file(config, channel, "") s.assert_any_call(ANY, "expected\n", "expected\n") @@ -284,6 +323,40 @@ def test_file_oracle_dont_strip_lines_correct( assert result.readable_actual == "expected\nexpected2\n" +def test_correct_error_actual_not_found(tmp_path: Path, pytestconfig: pytest.Config): + config = oracle_config( + tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} + ) + channel = OutputFileData( + content="Hallo world!", + path="expected.txt", + content_type=TextChannelType.TEXT, + ) + result = evaluate_file(config, channel, "") + assert result.result.enum == Status.RUNTIME_ERROR + assert ( + result.result.human == "File not found." + or result.result.human == "Bestand niet gevonden." + ) + + +def test_correct_error_expected_not_found(tmp_path: Path, pytestconfig: pytest.Config): + config = oracle_config( + tmp_path, pytestconfig, {"mode": "line", "stripNewlines": False} + ) + channel = OutputFileData( + content="actual.txt", + path="expected.txt", + content_type=TextChannelType.FILE, + ) + try: + evaluate_file(config, channel, "") + except ValueError: + print("As expected") + else: + assert False + + def test_exception_oracle_only_messages_correct( tmp_path: Path, pytestconfig: pytest.Config ): diff --git a/tests/test_suite.py b/tests/test_suite.py index d194691bb..4eff78c28 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -10,6 +10,7 @@ ExitCodeOutputChannel, FileOutputChannel, MainInput, + TextChannelType, TextData, TextOutputChannel, ValueOutputChannel, @@ -33,6 +34,13 @@ def test_text_output_is_compatible_oracle(): def test_file_output_is_compatible_oracle(): old_structure = { + "output_data": [ + { + "content_type": TextChannelType.TEXT, + "content": "some content", + "path": "output.py", + } + ], "evaluator": { "function": {"file": "evaluate.py"}, "type": "custom_check", @@ -108,13 +116,19 @@ def test_file_show_expected_is_accepted(): scheme = """ { "show_expected": true, - "expected_path": "hallo", - "actual_path": "hallo" + "output_data": [ + { + "content_type": "text", + "content": "hallo", + "path": "hallo.txt" + } + ] } """ result = get_converter().loads(scheme, FileOutputChannel) - assert result.expected_path == "hallo" - assert result.actual_path == "hallo" + + assert result.output_data[0].content == "hallo" + assert result.output_data[0].path == "hallo.txt" def test_value_show_expected_is_accepted(): diff --git a/tests/tested-draft7.json b/tests/tested-draft7.json index 1e49ec747..0f16450e0 100644 --- a/tests/tested-draft7.json +++ b/tests/tested-draft7.json @@ -28,7 +28,8 @@ "object", "string", "oracle", - "expression" + "expression", + "path" ] }, "stringArray": {