diff --git a/tested/configs.py b/tested/configs.py index 0cb0cc03a..5e58010fd 100644 --- a/tested/configs.py +++ b/tested/configs.py @@ -8,6 +8,7 @@ from attrs import define, evolve, field +from tested.dodona import ExtendedMessage from tested.parsing import fallback_field, get_converter from tested.testsuite import ExecutionMode, Suite, SupportedLanguage from tested.utils import get_identifier, smart_close @@ -129,6 +130,7 @@ class Bundle: language: "Language" global_config: GlobalConfig out: IO + messages: set[ExtendedMessage] = set() @property def config(self) -> DodonaConfig: @@ -207,6 +209,7 @@ def create_bundle( output: IO, suite: Suite, language: str | None = None, + messages: set[ExtendedMessage] | None = None, ) -> Bundle: """ Create a configuration bundle. @@ -216,7 +219,7 @@ def create_bundle( :param suite: The test suite. :param language: Optional programming language. If None, the one from the Dodona configuration will be used. - + :param messages: Messages generated out of the translate parser. :return: The configuration bundle. """ import tested.languages as langs @@ -232,4 +235,10 @@ def create_bundle( suite=suite, ) lang_config = langs.get_language(global_config, language) - return Bundle(language=lang_config, global_config=global_config, out=output) + + if messages is None: + messages = set() + + return Bundle( + language=lang_config, global_config=global_config, out=output, messages=messages + ) diff --git a/tested/descriptions/renderer.py b/tested/descriptions/renderer.py index 5690f12c4..2e766ac68 100644 --- a/tested/descriptions/renderer.py +++ b/tested/descriptions/renderer.py @@ -83,7 +83,7 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: rendered_dsl = self.render_children(element) # Parse the DSL - parsed_dsl = parse_dsl(rendered_dsl) + parsed_dsl = parse_dsl(rendered_dsl).data # Get all actual tests tests = [] @@ -95,7 +95,7 @@ def _render_dsl_statements(self, element: block.FencedCode) -> str: resulting_lines = [] prompt = self.bundle.language.get_declaration_metadata().get("prompt", ">") for testcase in tests: - stmt_message, _ = get_readable_input(self.bundle, testcase) + stmt_message = get_readable_input(self.bundle, testcase) resulting_lines.append(f"{prompt} {stmt_message.description}") output_lines = get_expected_output(self.bundle, testcase) resulting_lines.extend(output_lines) diff --git a/tested/dodona.py b/tested/dodona.py index 90e5f7683..3ece258a2 100644 --- a/tested/dodona.py +++ b/tested/dodona.py @@ -27,7 +27,7 @@ class Permission(StrEnum): ZEUS = auto() -@define +@define(frozen=True) class ExtendedMessage: description: str format: str = "text" @@ -36,10 +36,11 @@ class ExtendedMessage: @define class Metadata: - """Currently only used for the Python tutor""" + """Currently used for the Python tutor and rendering files in Dodona.""" statements: str | None stdin: str | None + files: list[dict[str, str]] | None Message = ExtendedMessage | str diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index d9e3f4074..2aafb2548 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -39,7 +39,14 @@ } ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +104,15 @@ "tab" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +164,15 @@ "unit" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +252,15 @@ "testcases" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +282,15 @@ "script" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -275,13 +314,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -335,7 +368,15 @@ } ] }, - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of input files used in the expressions, statements, arguments and descriptions of the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -354,6 +395,10 @@ "$ref" : "#/definitions/textOutputChannel" }, "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -371,13 +416,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -431,7 +470,7 @@ } ] }, - "files" : { + "input_files" : { "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -449,12 +488,65 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" } } }, + "stdinData" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": ["content"], + "additionalProperties": false, + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + }, + { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + } + ] + }, "expressionOrStatement" : { "oneOf" : [ { @@ -488,9 +580,9 @@ } ] }, - "file" : { + "deprecatedFile": { "type" : "object", - "description" : "A file used in the test suite.", + "description" : "A file used in the test suite (DEPRECATED).", "required" : [ "name", "url" @@ -498,12 +590,30 @@ "properties" : { "name" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "url" : { "type" : "string", "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + "file" : { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "path" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." + }, + "content" : { + "type" : "string", + "description" : "The actual content of the file." } } }, @@ -515,10 +625,28 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "data" + "oneOf": [ + { + "required" : [ + "data" + ] + }, + { + "required" : [ + "content" + ] + } ], + "additionalProperties" : false, "properties" : { + "content": { + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "data" : { "$ref" : "#/definitions/textualType" }, @@ -530,17 +658,96 @@ } } }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : ["path"], + "additionalProperties" : false, + "properties" : { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "additionalProperties" : false, + "oneOf": [ + { + "required" : [ + "oracle", + "file", + "data" + ] + }, + { + "required" : [ + "oracle", + "file", + "content" + ] + } + ], + "properties" : { + "content": { + "$ref" : "#/definitions/textualType" + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + }, { "type" : "object", "description" : "Custom oracle for text values.", + "additionalProperties" : false, "required" : [ "oracle", "file", - "data" + "path" ], "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, "oracle" : { "const" : "custom_check" @@ -572,7 +779,7 @@ } ] }, - "fileOutputChannel": { + "deprecatedFileOutputChannel": { "anyOf" : [ { "type" : "object", @@ -646,6 +853,144 @@ } ] }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "array", + "description" : "Built-in oracle for files.", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "data" + ], + "properties" : { + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "data", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "anyOf": [ + { + "type": "string" + }, + { + "type": "path" + } + ], + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, "returnOutputChannel" : { "oneOf" : [ { @@ -865,7 +1210,8 @@ "string", "number", "integer", - "boolean" + "boolean", + "path" ] }, "yamlValue" : { @@ -873,7 +1219,8 @@ "not" : { "type" : [ "oracle", - "expression" + "expression", + "path" ] } }, diff --git a/tested/dsl/schema.json b/tested/dsl/schema.json index f8d066bca..f9aa05648 100644 --- a/tested/dsl/schema.json +++ b/tested/dsl/schema.json @@ -39,7 +39,14 @@ } ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { "description" : "A list of files used in the test suite.", "type" : "array", "items" : { @@ -97,7 +104,15 @@ "tab" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -149,7 +164,15 @@ "unit" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -229,7 +252,15 @@ "testcases" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -251,7 +282,15 @@ "script" ], "properties" : { - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -275,13 +314,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -335,7 +368,15 @@ } ] }, - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -353,6 +394,14 @@ "description" : "Expected output at stdout", "$ref" : "#/definitions/textOutputChannel" }, + "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { + "description" : "Expected files generated by the submission.", + "$ref" : "#/definitions/fileOutputChannel" + }, "exit_code" : { "type" : "integer", "description" : "Expected exit code for the run" @@ -367,13 +416,7 @@ "$ref" : "#/definitions/message" }, "stdin" : { - "description" : "Stdin for this context", - "type" : [ - "string", - "number", - "integer", - "boolean" - ] + "$ref" : "#/definitions/stdinData" }, "arguments" : { "type" : "array", @@ -427,7 +470,15 @@ } ] }, - "files" : { + "files": { + "description" : "A list of files used in the test suite (DEPRECATED).", + "type" : "array", + "items" : { + "$ref" : "#/definitions/deprecatedFile" + } + }, + "input_files" : { + "description" : "A list of files used in the test suite.", "type" : "array", "items" : { "$ref" : "#/definitions/file" @@ -446,6 +497,10 @@ "$ref" : "#/definitions/textOutputChannel" }, "file": { + "description" : "Expected file generated by the submission (DEPRECATED).", + "$ref" : "#/definitions/deprecatedFileOutputChannel" + }, + "output_files": { "description" : "Expected files generated by the submission.", "$ref" : "#/definitions/fileOutputChannel" }, @@ -455,6 +510,51 @@ } } }, + "stdinData" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["content"], + "properties": { + "content": { + "type" : [ + "string", + "number", + "integer", + "boolean", + "object" + ], + "description": "The actual content that will be used for stdin." + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + }, + { + "type": "object", + "additionalProperties" : false, + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + } + } + } + ] + }, "expressionOrStatement" : { "oneOf" : [ { @@ -488,7 +588,7 @@ } ] }, - "file" : { + "deprecatedFile": { "type" : "object", "description" : "A file used in the test suite.", "required" : [ @@ -498,12 +598,30 @@ "properties" : { "name" : { "type" : "string", - "description" : "The filename, including the file extension." + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." }, "url" : { "type" : "string", "format" : "uri", - "description" : "Relative path to the file in the `description` folder of an exercise." + "description" : "Relative path to the file in the `evaluation` folder of an exercise." + } + } + }, + "file" : { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "path" + ], + "additionalProperties" : false, + "properties" : { + "path" : { + "type" : "string", + "description" : "The filename, including the file extension. It is also the relative path to a file in the working directory." + }, + "content" : { + "type" : "string", + "description" : "The actual content of the file." } } }, @@ -515,13 +633,31 @@ { "type" : "object", "description" : "Built-in oracle for text values.", - "required" : [ - "data" + "oneOf": [ + { + "required" : [ + "data" + ] + }, + { + "required" : [ + "content" + ] + } ], + "additionalProperties" : false, "properties" : { + "content": { + "$ref" : "#/definitions/textualType", + "description" : "Content or relative path to the file in the `evaluation` folder of an exercise." + }, "data" : { "$ref" : "#/definitions/textualType" }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, "oracle" : { "const" : "builtin" }, @@ -530,17 +666,96 @@ } } }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : ["path"], + "additionalProperties" : false, + "properties" : { + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "additionalProperties" : false, + "oneOf": [ + { + "required" : [ + "oracle", + "file", + "data" + ] + }, + { + "required" : [ + "oracle", + "file", + "content" + ] + } + ], + "properties" : { + "content": { + "$ref" : "#/definitions/textualType" + }, + "data" : { + "$ref" : "#/definitions/textualType" + }, + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + }, { "type" : "object", "description" : "Custom oracle for text values.", + "additionalProperties" : false, "required" : [ "oracle", "file", - "data" + "path" ], "properties" : { - "data" : { - "$ref" : "#/definitions/textualType" + "path": { + "type": "string", + "description": "The path that will be shown in the feedback. It also provides the path to the content when content isn't specified." }, "oracle" : { "const" : "custom_check" @@ -572,7 +787,7 @@ } ] }, - "fileOutputChannel": { + "deprecatedFileOutputChannel": { "anyOf" : [ { "type" : "object", @@ -646,6 +861,123 @@ } ] }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "array", + "description" : "Built-in oracle for files.", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "data" + ], + "properties" : { + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "data", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "data": { + "type": "array", + "items" : { + "type" : "object", + "required" : [ + "path", + "content" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Expected content for the file or path to file with the expected content for the file, relative to the evaluation directory." + }, + "path" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + } + } + } + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, "returnOutputChannel" : { "oneOf" : [ { diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 245b0cf7d..101b1c4c4 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -1,4 +1,5 @@ import json +import os import sys import textwrap from collections.abc import Callable @@ -31,7 +32,7 @@ StringTypes, resolve_to_basic, ) -from tested.dodona import ExtendedMessage +from tested.dodona import ExtendedMessage, Permission from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string from tested.parsing import get_converter, suite_to_json from tested.serialisation import ( @@ -53,23 +54,25 @@ ExitCodeOutputChannel, ExpectedException, FileOutputChannel, - FileUrl, GenericTextOracle, IgnoredChannel, + InputFile, LanguageLiterals, LanguageSpecificOracle, MainInput, Output, + OutputFileData, Suite, SupportedLanguage, Tab, Testcase, TextBuiltin, + TextChannelType, TextData, TextOutputChannel, ValueOutputChannel, ) -from tested.utils import get_args, recursive_dict_merge +from tested.utils import DataWithMessage, get_args, recursive_dict_merge YamlDict = dict[str, "YamlObject"] @@ -84,13 +87,26 @@ class ExpressionString(str): pass +class PathString(str): + pass + + class ReturnOracle(dict): pass OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | PathString ) @@ -130,6 +146,12 @@ def _expression_string(loader: yaml.Loader, node: yaml.Node) -> ExpressionString return ExpressionString(result) +def _path_string(loader: yaml.Loader, node: yaml.Node) -> PathString: + result = _parse_yaml_value(loader, node) + assert isinstance(result, str), f"A path must be a string, got {result}" + return PathString(result) + + def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: result = _parse_yaml_value(loader, node) assert isinstance( @@ -148,6 +170,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor("!path", _path_string, loader) try: return yaml.load(yaml_stream, loader) @@ -187,6 +210,10 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_path(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, PathString) + + def load_schema_validator( dsl_object: YamlObject = None, file: str = "schema-strict.json" ) -> Validator: @@ -215,9 +242,11 @@ def validate_tested_dsl_expression(value: object) -> bool: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression) + type_checker = ( + original_validator.TYPE_CHECKER.redefine("oracle", is_oracle) + .redefine("expression", is_expression) + .redefine("path", is_path) + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)( validate_tested_dsl_expression @@ -246,26 +275,46 @@ class DslContext: - The "files" property, which is a list of files. """ - files: list[FileUrl] = field(factory=list) + files: list[InputFile] = field(factory=list) config: dict[str, dict] = field(factory=dict) language: SupportedLanguage | Literal["tested"] = "tested" - def deepen_context(self, new_level: YamlDict | None) -> "DslContext": + def deepen_context( + self, new_level: YamlDict | None, workdir: Path | None + ) -> DataWithMessage["DslContext"]: """ Merge certain fields of the new object with the current context, resulting in a new context for the new level. :param new_level: The new object from the DSL to get information from. + :param workdir: The working directory where all files are located. :return: A new context. """ if new_level is None: - return self + return DataWithMessage(data=self, messages=set()) + deprecated_usage = set() the_files = self.files - if "files" in new_level: - assert isinstance(new_level["files"], list) - additional_files = {_convert_file(f) for f in new_level["files"]} + if "input_files" in new_level or "files" in new_level: + key = "input_files" + if "files" in new_level: + key = "files" + deprecated_usage.add( + ExtendedMessage( + f"WARNING: You are using YAML syntax to specify input files with the key 'files'. This usage is deprecated! Try using 'input_files' instead.", + permission=Permission.STAFF, + ) + ) + + files = new_level[key] + assert isinstance(files, list) + additional_files = { + _convert_input_file( + f, workdir=workdir, not_deprecated_usage=len(deprecated_usage) == 0 + ) + for f in files + } the_files = list(set(self.files) | additional_files) the_config = self.config @@ -273,7 +322,10 @@ def deepen_context(self, new_level: YamlDict | None) -> "DslContext": assert isinstance(new_level["config"], dict) the_config = recursive_dict_merge(the_config, new_level["config"]) - return evolve(self, files=the_files, config=the_config) + return DataWithMessage( + data=evolve(self, files=the_files, config=the_config), + messages=deprecated_usage, + ) def merge_inheritable_with_specific_config( self, level: YamlDict, config_name: str @@ -402,10 +454,26 @@ def _convert_value(value: YamlObject) -> Value: return _tested_type_to_value(tested_type) -def _convert_file(link_file: YamlDict) -> FileUrl: - assert isinstance(link_file["name"], str) - assert isinstance(link_file["url"], str) - return FileUrl(name=link_file["name"], url=link_file["url"]) +def _convert_input_file( + input_file: YamlDict, workdir: Path | None, not_deprecated_usage: bool +) -> InputFile: + path_key = "path" if not_deprecated_usage else "name" + path_str = input_file[path_key] + assert isinstance(path_str, str) + + content = "" + if "content" in input_file: + content = input_file["content"] + assert isinstance(content, str) + if workdir is not None: + full_path = workdir / path_str + dir_name = os.path.dirname(full_path) + if dir_name: + os.makedirs(os.path.dirname(full_path), exist_ok=True) + with open(full_path, "w", encoding="utf-8") as f: + f.write(content) + + return InputFile(path=path_str, content=content) def _convert_evaluation_function(stream: dict) -> EvaluationFunction: @@ -454,42 +522,57 @@ def _convert_text_output_channel( ) -> TextOutputChannel: # Get the config applicable to this level. # Either attempt to get it from an object, or using the inherited options as is. + path = None + data = None if isinstance(stream, str): config = context.config.get(config_name, dict()) - raw_data = stream + data = stream else: assert isinstance(stream, dict) config = context.merge_inheritable_with_specific_config(stream, config_name) - raw_data = str(stream["data"]) + if "path" in stream: + path = str(stream["path"]) + + if "content" in stream or "data" in stream: + data = str(stream.get("content", stream.get("data"))) + + assert path or data # Normalize the data if necessary. - if config.get("normalizeTrailingNewlines", True): - data = _ensure_trailing_newline(raw_data) + if config.get("normalizeTrailingNewlines", True) and path is None: + assert data is not None + data = _ensure_trailing_newline(str(data)) + + if path is not None: + text_output = TextOutputChannel(data=data, path=path, type=TextChannelType.FILE) else: - data = raw_data + text_output = TextOutputChannel(data=data) if isinstance(stream, str): - return TextOutputChannel(data=data, oracle=GenericTextOracle(options=config)) + text_output.oracle = GenericTextOracle(options=config) + return text_output else: assert isinstance(stream, dict) if "oracle" not in stream or stream["oracle"] == "builtin": - return TextOutputChannel( - data=data, oracle=GenericTextOracle(options=config) - ) + + text_output.oracle = GenericTextOracle(options=config) + return text_output elif stream["oracle"] == "custom_check": - return TextOutputChannel( - data=data, oracle=_convert_custom_check_oracle(stream) - ) + text_output.oracle = _convert_custom_check_oracle(stream) + return text_output raise TypeError(f"Unknown text oracle type: {stream['oracle']}") -def _convert_file_output_channel( +def _convert_file_output_channel_deprecated( stream: YamlObject, context: DslContext, config_name: str ) -> FileOutputChannel: assert isinstance(stream, dict) - expected = str(stream["content"]) - actual = str(stream["location"]) + data = OutputFileData( + content_type=TextChannelType.TEXT, + content=str(stream["content"]), + path=str(stream["location"]), + ) if "oracle" not in stream or stream["oracle"] == "builtin": config = context.merge_inheritable_with_specific_config(stream, config_name) @@ -501,19 +584,70 @@ def _convert_file_output_channel( "line", ), f"The file oracle only supports modes full and line, not {config['mode']}" return FileOutputChannel( - expected_path=expected, - actual_path=actual, + output_data=[data], oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), ) elif stream["oracle"] == "custom_check": return FileOutputChannel( - expected_path=expected, - actual_path=actual, + output_data=[data], oracle=_convert_custom_check_oracle(stream), ) raise TypeError(f"Unknown file oracle type: {stream['oracle']}") +def _convert_output_file_channel( + output_files: YamlObject, context: DslContext, config_name: str +) -> FileOutputChannel: + + file_data = [] + data = output_files + if isinstance(output_files, dict): + data = output_files["data"] + assert isinstance(data, list) + + for item in data: + assert isinstance(item, dict) + content = item["content"] + if isinstance(content, PathString): + content_type = TextChannelType.FILE + else: + content = str(content) + content_type = TextChannelType.TEXT + + file_data.append( + OutputFileData( + content_type=content_type, + content=content, + path=str(item["path"]), + ) + ) + + if ( + not isinstance(output_files, dict) + or "oracle" not in output_files + or output_files["oracle"] == "builtin" + ): + level = {} if not isinstance(output_files, dict) else output_files + config = context.merge_inheritable_with_specific_config(level, config_name) + if "mode" not in config: + config["mode"] = "full" + + assert config["mode"] in ( + "full", + "line", + ), f"The file oracle only supports modes full and line, not {config['mode']}" + return FileOutputChannel( + output_data=file_data, + oracle=GenericTextOracle(name=TextBuiltin.FILE, options=config), + ) + elif output_files["oracle"] == "custom_check": + return FileOutputChannel( + output_data=file_data, + oracle=_convert_custom_check_oracle(output_files), + ) + raise TypeError(f"Unknown file oracle type: {output_files['oracle']}") + + def _convert_yaml_value(stream: YamlObject) -> Value | None: if isinstance(stream, ExpressionString): # We have an expression string. @@ -568,8 +702,12 @@ def _validate_testcase_combinations(testcase: YamlDict): raise ValueError("A statement cannot have an expected return value.") -def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: - context = context.deepen_context(testcase) +def _convert_testcase( + testcase: YamlDict, context: DslContext, workdir: Path | None +) -> DataWithMessage[Testcase]: + context_with_data = context.deepen_context(testcase, workdir) + context = context_with_data.data + deprecated_messages = context_with_data.messages # This is backwards compatability to some extend. # TODO: remove this at some point. @@ -577,6 +715,7 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: testcase["expression"] = testcase.pop("statement") line_comment = "" + stdin_file = None _validate_testcase_combinations(testcase) if (expr_stmt := testcase.get("statement", testcase.get("expression"))) is not None: if isinstance(expr_stmt, dict) or context.language != "tested": @@ -599,8 +738,30 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: return_channel = IgnoredChannel.IGNORED if "statement" in testcase else None else: if "stdin" in testcase: - assert isinstance(testcase["stdin"], str) - stdin = TextData(data=_ensure_trailing_newline(testcase["stdin"])) + stdin_data = testcase["stdin"] + data = None + path = "" + if isinstance(stdin_data, str): + data = _ensure_trailing_newline(stdin_data) + else: + assert isinstance(stdin_data, dict) + if "path" in stdin_data: + path = stdin_data["path"] + assert isinstance(path, str) + + if "content" in stdin_data: + content = stdin_data["content"] + assert isinstance(content, str) + data = _ensure_trailing_newline(content) + + if path: + stdin_file = InputFile( + path=path, + content=data if data is not None else "", + ) + stdin = TextData(data=data, path=path, type=TextChannelType.FILE) + else: + stdin = TextData(data=data) else: stdin = EmptyChannel.NONE arguments = testcase.get("arguments", []) @@ -616,7 +777,15 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: if (stdout := testcase.get("stdout")) is not None: output.stdout = _convert_text_output_channel(stdout, context, "stdout") if (file := testcase.get("file")) is not None: - output.file = _convert_file_output_channel(file, context, "file") + output.file = _convert_file_output_channel_deprecated(file, context, "file") + deprecated_messages.add( + ExtendedMessage( + "WARNING: You are using YAML syntax to specify output files with the key 'file'. This usage is deprecated! Try using 'output_files' instead.", + permission=Permission.STAFF, + ) + ) + if (file := testcase.get("output_files")) is not None: + output.file = _convert_output_file_channel(file, context, "output_files") if (stderr := testcase.get("stderr")) is not None: output.stderr = _convert_text_output_channel(stderr, context, "stderr") if (exception := testcase.get("exception")) is not None: @@ -660,24 +829,43 @@ def _convert_testcase(testcase: YamlDict, context: DslContext) -> Testcase: else: the_description = None - return Testcase( - description=the_description, - input=the_input, - output=output, - link_files=context.files, - line_comment=line_comment, + input_files = context.files + if stdin_file is not None: + input_files = list(set(input_files) | {stdin_file}) + + return DataWithMessage( + data=Testcase( + description=the_description, + input=the_input, + output=output, + input_files=input_files, + line_comment=line_comment, + ), + messages=deprecated_messages, ) -def _convert_context(context: YamlDict, dsl_context: DslContext) -> Context: - dsl_context = dsl_context.deepen_context(context) +def _convert_context( + context: YamlDict, dsl_context: DslContext, workdir: Path | None +) -> DataWithMessage[Context]: + dsl_context_with_messages = dsl_context.deepen_context(context, workdir) + dsl_context = dsl_context_with_messages.data + deprecated_messages = dsl_context_with_messages.messages raw_testcases = context.get("script", context.get("testcases")) assert isinstance(raw_testcases, list) - testcases = _convert_dsl_list(raw_testcases, dsl_context, _convert_testcase) - return Context(testcases=testcases) + testcases = _convert_dsl_list( + raw_testcases, dsl_context, workdir, _convert_testcase + ) + deprecated_messages.update(testcases.messages) + return DataWithMessage( + data=Context(testcases=testcases.data), + messages=deprecated_messages, + ) -def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: +def _convert_tab( + tab: YamlDict, context: DslContext, workdir: Path | None +) -> DataWithMessage[Tab]: """ Translate a DSL tab to a full test suite tab. @@ -685,51 +873,81 @@ def _convert_tab(tab: YamlDict, context: DslContext) -> Tab: :param context: The context with config for the parent level. :return: A full tab. """ - context = context.deepen_context(tab) + context_with_messages = context.deepen_context(tab, workdir) + context = context_with_messages.data + deprecated_messages = context_with_messages.messages name = tab.get("unit", tab.get("tab")) assert isinstance(name, str) # The tab can have testcases or contexts. if "contexts" in tab: assert isinstance(tab["contexts"], list) - contexts = _convert_dsl_list(tab["contexts"], context, _convert_context) + contexts_with_messages = _convert_dsl_list( + tab["contexts"], context, workdir, _convert_context + ) + contexts = contexts_with_messages.data + deprecated_messages.update(contexts_with_messages.messages) elif "cases" in tab: assert "unit" in tab # We have testcases N.S. / contexts O.S. assert isinstance(tab["cases"], list) - contexts = _convert_dsl_list(tab["cases"], context, _convert_context) + contexts_with_messages = _convert_dsl_list( + tab["cases"], context, workdir, _convert_context + ) + contexts = contexts_with_messages.data + deprecated_messages.update(contexts_with_messages.messages) elif "testcases" in tab: # We have scripts N.S. / testcases O.S. assert "tab" in tab assert isinstance(tab["testcases"], list) - testcases = _convert_dsl_list(tab["testcases"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in testcases] + testcases = _convert_dsl_list( + tab["testcases"], context, workdir, _convert_testcase + ) + deprecated_messages.update(testcases.messages) + contexts = [Context(testcases=[t]) for t in testcases.data] else: assert "scripts" in tab assert isinstance(tab["scripts"], list) - testcases = _convert_dsl_list(tab["scripts"], context, _convert_testcase) - contexts = [Context(testcases=[t]) for t in testcases] + testcases = _convert_dsl_list( + tab["scripts"], context, workdir, _convert_testcase + ) + deprecated_messages.update(testcases.messages) + contexts = [Context(testcases=[t]) for t in testcases.data] - return Tab(name=name, contexts=contexts) + return DataWithMessage( + data=Tab(name=name, contexts=contexts), + messages=deprecated_messages, + ) T = TypeVar("T") def _convert_dsl_list( - dsl_list: list, context: DslContext, converter: Callable[[YamlDict, DslContext], T] -) -> list[T]: + dsl_list: list, + context: DslContext, + workdir: Path | None, + converter: Callable[[YamlDict, DslContext, Path | None], DataWithMessage[T]], +) -> DataWithMessage[list[T]]: """ Convert a list of YAML objects into a test suite object. """ objects = [] + deprecated_messages = set() for dsl_object in dsl_list: assert isinstance(dsl_object, dict) - objects.append(converter(dsl_object, context)) - return objects + ob = converter(dsl_object, context, workdir) + deprecated_messages.update(ob.messages) + objects.append(ob.data) + return DataWithMessage( + data=objects, + messages=deprecated_messages, + ) -def _convert_dsl(dsl_object: YamlObject) -> Suite: +def _convert_dsl( + dsl_object: YamlObject, workdir: Path | None +) -> DataWithMessage[Suite]: """ Translate a DSL test suite into a full test suite. @@ -740,38 +958,51 @@ def _convert_dsl(dsl_object: YamlObject) -> Suite: :return: A full test suite. """ context = DslContext() + deprecated_messages = set() + if isinstance(dsl_object, list): namespace = None tab_list = dsl_object else: assert isinstance(dsl_object, dict) namespace = dsl_object.get("namespace") - context = context.deepen_context(dsl_object) + context_with_messages = context.deepen_context(dsl_object, workdir) + context = context_with_messages.data + deprecated_messages.update(context_with_messages.messages) tab_list = dsl_object.get("units", dsl_object.get("tabs")) assert isinstance(tab_list, list) if (language := dsl_object.get("language", "tested")) != "tested": language = SupportedLanguage(language) context = evolve(context, language=language) - tabs = _convert_dsl_list(tab_list, context, _convert_tab) + tabs = _convert_dsl_list(tab_list, context, workdir, _convert_tab) + deprecated_messages.update(tabs.messages) if namespace: assert isinstance(namespace, str) - return Suite(tabs=tabs, namespace=namespace) + return DataWithMessage( + data=Suite(tabs=tabs.data, namespace=namespace), + messages=deprecated_messages, + ) else: - return Suite(tabs=tabs) + return DataWithMessage( + data=Suite(tabs=tabs.data), + messages=deprecated_messages, + ) -def parse_dsl(dsl_string: str) -> Suite: +def parse_dsl(dsl_string: str, workdir: Path | None = None) -> DataWithMessage[Suite]: """ Parse a string containing a DSL test suite into our representation, a test suite. :param dsl_string: The string containing a DSL. + :param workdir: The working directory for the test suite. :return: The parsed and converted test suite. """ dsl_object = _parse_yaml(dsl_string) _validate_dsl(dsl_object) - return _convert_dsl(dsl_object) + + return _convert_dsl(dsl_object, workdir) def translate_to_test_suite(dsl_string: str) -> str: @@ -779,7 +1010,8 @@ def translate_to_test_suite(dsl_string: str) -> str: Convert a DSL to a test suite. :param dsl_string: The DSL. + :param workdir: The working directory for the test suite. :return: The test suite. """ - suite = parse_dsl(dsl_string) - return suite_to_json(suite) + suite = parse_dsl(dsl_string, Path(".")) + return suite_to_json(suite.data) diff --git a/tested/judge/core.py b/tested/judge/core.py index 24bdd3b31..bddaaeb0e 100644 --- a/tested/judge/core.py +++ b/tested/judge/core.py @@ -116,6 +116,9 @@ def judge(bundle: Bundle): # Do the set-up for the judgement. collector = OutputManager(bundle.out) collector.add(StartJudgement()) + if bundle.messages: + collector.add_messages(bundle.messages) + max_time = float(bundle.config.time_limit) * 0.9 start = time.perf_counter() @@ -338,27 +341,35 @@ def _process_results( compilation_results=compilation_results, ) + input_files = [] + for case in planned.context.testcases: + for file in case.input_files: + file_data = {"path": file.path} + if file.content != "": + file_data["content"] = file.content + input_files.append(file_data) + if not input_files: + input_files = None + if bundle.language.supports_debug_information(): - # TODO: this is currently very Python-specific - # See if we need a callback to the language modules in the future. - # TODO: we could probably re-use the "readable_input" function here, - # since it only differs a bit. meta_statements = [] meta_stdin = None + for case in planned.context.testcases: if case.is_main_testcase(): assert isinstance(case.input, MainInput) - if isinstance(case.input.stdin, TextData): - meta_stdin = case.input.stdin.get_data_as_string( - bundle.config.resources - ) + if ( + isinstance(case.input.stdin, TextData) + and case.input.stdin.data is not None + ): + meta_stdin = case.input.stdin.data elif isinstance(case.input, Statement): stmt = generate_statement(bundle, case.input) meta_statements.append(stmt) elif isinstance(case.input, LanguageLiterals): stmt = case.input.get_for(bundle.config.programming_language) meta_statements.append(stmt) - else: + elif not case.is_main_testcase(): raise AssertionError(f"Found unknown case input type: {case.input}") if meta_statements: @@ -367,17 +378,24 @@ def _process_results( # Don't add empty statements meta_statements = None + if not input_files: + input_files = None + collector.add( CloseContext( data=Metadata( - statements=meta_statements, - stdin=meta_stdin, + statements=meta_statements, files=input_files, stdin=meta_stdin ) ), planned.context_index, ) else: - collector.add(CloseContext(), planned.context_index) + collector.add( + CloseContext( + data=Metadata(files=input_files, stdin=None, statements=None) + ), + planned.context_index, + ) if continue_ in (Status.TIME_LIMIT_EXCEEDED, Status.MEMORY_LIMIT_EXCEEDED): return continue_, currently_open_tab diff --git a/tested/judge/evaluation.py b/tested/judge/evaluation.py index 44b2571ab..f64aa8624 100644 --- a/tested/judge/evaluation.py +++ b/tested/judge/evaluation.py @@ -1,6 +1,4 @@ -import html import logging -from collections.abc import Collection from enum import StrEnum, unique from pathlib import Path from typing import Literal @@ -42,9 +40,9 @@ ExitCodeOutputChannel, FileOutput, FileOutputChannel, - FileUrl, IgnoredChannel, OutputChannel, + OutputFileData, SpecialOutputChannel, Testcase, TextOutput, @@ -109,40 +107,54 @@ def _evaluate_channel( evaluator = get_oracle( bundle, context_directory, output, testcase, unexpected_status=unexpected_status ) - # Run the oracle. - evaluation_result = evaluator(output, actual if actual else "") - status = evaluation_result.result - - # Decide if we should show this channel or not. - is_correct = status.enum == Status.CORRECT - should_report_case = should_show(output, channel, evaluation_result) - - if not should_report_case and is_correct: - # We do report that a test is correct, to set the status. - return False - - expected = evaluation_result.readable_expected - out.add(StartTest(expected=expected, channel=channel)) - - # Report any messages we received. - for message in evaluation_result.messages: - out.add(AppendMessage(message=message)) - - missing = False - if actual is None: - out.add(AppendMessage(message=get_i18n_string("judge.evaluation.missing"))) - missing = True - elif should_report_case and timeout and not is_correct: - status.human = get_i18n_string("judge.evaluation.time-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - elif should_report_case and memory and not is_correct: - status.human = get_i18n_string("judge.evaluation.memory-limit") - status.enum = Status.TIME_LIMIT_EXCEEDED - out.add(AppendMessage(message=status.human)) - - # Close the test. - out.add(CloseTest(generated=evaluation_result.readable_actual, status=status)) + + if channel == Channel.FILE and output != "ignored": + assert isinstance(output, FileOutputChannel) + output_channels = output.output_data + else: + output_channels = [output] + + missing = actual is None + + for output_element in output_channels: + # Run the oracle. + evaluation_result = evaluator(output_element, actual if actual else "") + status = evaluation_result.result + + # Decide if we should show this channel or not. + is_correct = status.enum == Status.CORRECT + should_report_case = should_show(output, channel, evaluation_result) + + if should_report_case or not is_correct: + expected = evaluation_result.readable_expected + expected_channel = channel + if isinstance(output_element, OutputFileData): + expected_channel = f"file: {output_element.path}" + out.add(StartTest(expected=expected, channel=expected_channel)) + + # Report any messages we received. + for message in evaluation_result.messages: + out.add(AppendMessage(message=message)) + + if actual is None: + out.add( + AppendMessage(message=get_i18n_string("judge.evaluation.missing")) + ) + elif should_report_case and timeout and not is_correct: + status.human = get_i18n_string("judge.evaluation.time-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + elif should_report_case and memory and not is_correct: + status.human = get_i18n_string("judge.evaluation.memory-limit") + status.enum = Status.TIME_LIMIT_EXCEEDED + out.add(AppendMessage(message=status.human)) + + # Close the test. + out.add( + CloseTest(generated=evaluation_result.readable_actual, status=status) + ) + else: + missing = False return missing @@ -241,15 +253,11 @@ def evaluate_context_results( ) ) - # All files that will be used in this context. - all_files = context.get_files() - # Begin processing the normal testcases. for i, testcase in enumerate(context.testcases): _logger.debug(f"Evaluating testcase {i}") - readable_input, seen = get_readable_input(bundle, testcase) - all_files = all_files - seen + readable_input = get_readable_input(bundle, testcase) t_col = TestcaseCollector(StartTestcase(description=readable_input)) # Get the functions @@ -351,10 +359,6 @@ def evaluate_context_results( t_col.to_manager(collector, CloseTestcase(), i) - # Add file links - if all_files: - collector.add(_link_files_message(all_files)) - if exec_results.timeout: return Status.TIME_LIMIT_EXCEEDED if exec_results.memory: @@ -362,20 +366,6 @@ def evaluate_context_results( return None -def _link_files_message(link_files: Collection[FileUrl]) -> AppendMessage: - link_list = ", ".join( - f'' - f'{html.escape(link_file.name)}' - for link_file in link_files - ) - file_list_str = get_i18n_string( - "judge.evaluation.files", count=len(link_files), files=link_list - ) - description = f"
{file_list_str}
",
- "