diff --git a/tested/dsl/multilingual-schema.json b/tested/dsl/multilingual-schema.json index b32f98d4..f5dcf721 100644 --- a/tested/dsl/multilingual-schema.json +++ b/tested/dsl/multilingual-schema.json @@ -15,133 +15,269 @@ ], "definitions": { "_rootObject": { - "type": "object", "oneOf": [ { - "required": [ - "tabs" - ], - "not": { - "required": [ - "units" - ] - } - }, - { - "required": [ - "units" - ], - "not": { - "required": [ - "tabs" - ] - } - } - ], - "properties": { - "files": { + "type": "object", "oneOf": [ { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" + "required": [ + "tabs" + ], + "not": { + "required": [ + "units" + ] } }, { - "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, + "required": [ + "units" + ], "not": { - "anyOf": [ - { - "required": [ - "description" + "required": [ + "tabs" + ] + } + } + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + "namespace": { + "oneOf": [ + { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" }, - { - "required": [ - "value" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + "tabs": { + "$ref": "#/definitions/_tabList" + }, + "units": { + "$ref": "#/definitions/_unitList" + }, + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ + { + "$ref": "#/subDefinitions/programmingLanguage" + }, + { + "const": "tested" + } + ] + }, + "definitions": { + "oneOf": [ + { + "description": "Define hashes to use elsewhere.", + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "description": "Define hashes to use elsewhere.", + "type": "object" }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } - } - ] - }, - "namespace": { - "oneOf": [ - { - "type": "string", - "description": "Namespace of the submitted solution, in `snake_case`" + } + ] }, - { + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { "type": "object", + "description": "Define templates.", "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "oneOf": [ + { + "required": [ + "tabs" + ], + "not": { + "required": [ + "units" + ] + } + }, + { + "required": [ + "units" + ], + "not": { + "required": [ + "tabs" + ] + } + } + ], + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "namespace": { "type": "string", "description": "Namespace of the submitted solution, in `snake_case`" }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, + "tabs": { + "$ref": "#/definitions/_tabList" + }, + "units": { + "$ref": "#/definitions/_unitList" + }, + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ { - "required": [ - "value" - ] + "$ref": "#/subDefinitions/programmingLanguage" }, { - "required": [ - "types" - ] + "const": "tested" } ] + }, + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } } } - ] - }, - "tabs": { - "$ref": "#/definitions/_tabList" - }, - "units": { - "$ref": "#/definitions/_unitList" - }, - "language": { - "description": "Indicate that all code is in a specific language.", - "oneOf": [ - { - "$ref": "#/subDefinitions/programmingLanguage" - }, - { - "const": "tested" - } - ] - }, - "definitions": { - "description": "Define hashes to use elsewhere.", - "type": "object" - }, - "config": { - "$ref": "#/subDefinitions/inheritableConfigObject" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } } - } + ] }, "_tabList": { "oneOf": [ @@ -224,342 +360,275 @@ ] }, "tab": { - "type": "object", - "description": "A tab in the test suite.", - "required": [ - "tab" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" + "oneOf": [ + { + "type": "object", + "description": "A tab in the test suite.", + "required": [ + "tab" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + "hidden": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + } + ] + }, + "tab": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" }, - { - "required": [ - "value" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + "definitions": { + "oneOf": [ + { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "description": "Define objects to use elsewhere.", + "type": "object" }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] + } + ] + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" } } - ] - }, - "hidden": { - "type": "boolean", - "description": "Defines if the unit/tab is hidden for the student or not" - }, - "tab": { + }, "oneOf": [ { - "description": "The name of this tab.", - "type": "string" + "required": [ + "contexts" + ], + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" + } + } }, { - "type": "object", - "additionalProperties": { - "description": "The name of this tab.", - "type": "string" - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] + "required": [ + "testcases" + ], + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" + } } } ] }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" - }, - "config": { - "$ref": "#/subDefinitions/inheritableConfigObject" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." - } - }, - "oneOf": [ - { - "required": [ - "contexts" - ], - "properties": { - "contexts": { - "$ref": "#/definitions/_contextList" - } - } - }, { - "required": [ - "testcases" - ], - "properties": { - "testcases": { - "$ref": "#/definitions/_testcaseList" - } - } - } - ] - }, - "unit": { - "type": "object", - "description": "A unit in the test suite.", - "required": [ - "unit" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A tab in the test suite.", + "required": [ + "tab" + ], + "properties": { + "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { "$ref": "#/subDefinitions/file" } }, - "not": { + "hidden": { "anyOf": [ { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "The key of the parameter.", + "type": "string" }, { - "required": [ - "types" - ] + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" } ] - } - } - ] - }, - "hidden": { - "type": "boolean", - "description": "Defines if the unit/tab is hidden for the student or not" - }, - "unit": { - "oneOf": [ - { - "description": "The name of this tab.", - "type": "string" - }, - { - "type": "object", - "additionalProperties": { + }, + "tab": { "description": "The name of this tab.", "type": "string" }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" - }, - "config": { - "$ref": "#/subDefinitions/inheritableConfigObject" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." - } - }, - "oneOf": [ - { - "required": [ - "cases" - ], - "properties": { - "cases": { - "$ref": "#/definitions/_caseList" - } - } - }, - { - "required": [ - "scripts" - ], - "properties": { - "scripts": { - "$ref": "#/definitions/_scriptList" - } - } - } - ] - }, - "_contextList": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/context" - } - }, - { - "type": "object", - "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/context" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" }, - { - "required": [ - "value" - ] + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "_caseList": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/case" - } - }, - { - "type": "object", - "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/case" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] + "translations": { + "type": "object", + "description": "Define translations." }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + }, + "oneOf": [ { "required": [ - "value" - ] + "contexts" + ], + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" + } + } }, { "required": [ - "types" - ] + "testcases" + ], + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" + } + } } ] - } - } - ] - }, - "_testcaseList": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/testcase" - } - }, - { - "type": "object", - "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/testcase" - } }, "not": { "anyOf": [ @@ -583,644 +652,2108 @@ } ] }, - "_scriptList": { + "unit": { "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/script" - } - }, { "type": "object", - "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/script" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "context": { - "type": "object", - "description": "A set of testcase in the same context.", - "required": [ - "testcases" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "A unit in the test suite.", + "required": [ + "unit" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } - } - ] - }, - "context": { - "oneOf": [ - { - "type": "string", - "description": "Description of this context." + } + ] }, - { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Description of this context." - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] + "hidden": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + } + ] + }, + "unit": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" }, - { - "required": [ - "value" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + "definitions": { + "oneOf": [ + { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + { + "type": "object", + "additionalProperties": { + "description": "Define objects to use elsewhere.", + "type": "object" }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] + } + ] + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" } } - ] - }, - "testcases": { - "$ref": "#/definitions/_testcaseList" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." - } - } - }, - "case": { - "type": "object", - "description": "A test case.", - "required": [ - "script" - ], - "properties": { - "files": { + }, "oneOf": [ { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" + "required": [ + "cases" + ], + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" + } } }, { - "type": "object", - "additionalProperties": { + "required": [ + "scripts" + ], + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" + } + } + } + ] + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ + "unit" + ], + "properties": { + "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { "$ref": "#/subDefinitions/file" } }, - "not": { + "hidden": { "anyOf": [ { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "The key of the parameter.", + "type": "string" }, { - "required": [ - "types" - ] + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" } ] + }, + "unit": { + "description": "The name of this tab.", + "type": "string" + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } } - } - ] - }, - "context": { - "oneOf": [ - { - "type": "string", - "description": "Description of this context." }, - { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Description of this context." + "oneOf": [ + { + "required": [ + "cases" + ], + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" + } + } }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] + { + "required": [ + "scripts" + ], + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } + } + } + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" ] } - } - ] - }, - "script": { - "$ref": "#/definitions/_scriptList" + ] + } + } + ] + }, + "_contextList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } }, - "translations": { + { "type": "object", - "description": "Define translations in the global scope." + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } } - } + ] }, - "testcase": { - "type": "object", - "description": "An individual test for a statement or expression", - "additionalProperties": false, - "properties": { - "description": { - "$ref": "#/subDefinitions/message" + "_caseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } }, - "stdin": { - "oneOf": [ - { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "additionalProperties": { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" ] }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" ] } - } - ] + ] + } + } + ] + }, + "_testcaseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } }, - "arguments": { - "oneOf": [ - { - "description": "Array of program call arguments", - "type": "array", - "items": { - "oneOf": [ - { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "additionalProperties": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" ] - } - }, - { - "type": "object", - "additionalProperties": { - "description": "Array of program call arguments", - "type": "array", - "items": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } + { + "required": [ + "value" ] - } - } - ] - }, - "statement": { - "description": "The statement to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "expression": { - "description": "The expression to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "exception": { - "description": "Expected exception message", - "$ref": "#/subDefinitions/exceptionChannel" - }, - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } + { + "required": [ + "types" ] } - } - ] - }, - "return": { - "description": "Expected return value", - "$ref": "#/subDefinitions/returnOutputChannel" - }, - "stderr": { - "description": "Expected output at stderr", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "stdout": { - "description": "Expected output at stdout", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "file": { - "description": "Expected files generated by the submission.", - "$ref": "#/subDefinitions/fileOutputChannel" - }, - "exit_code": { - "type": "integer", - "description": "Expected exit code for the run" + ] + } } - } + ] }, - "script": { - "type": "object", - "description": "An individual test (script) for a statement or expression", - "properties": { - "description": { - "$ref": "#/subDefinitions/message" + "_scriptList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } }, - "stdin": { - "oneOf": [ - { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "additionalProperties": { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" + { + "type": "object", + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" ] }, - "not": { + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ + "testcases" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "testcases": { + "$ref": "#/definitions/_testcaseList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ + "testcases" + ], + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "context": { + "type": "string", + "description": "Description of this context." + }, + "testcases": { + "$ref": "#/definitions/_testcaseList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "case": { + "oneOf": [ + { + "type": "object", + "description": "A test case.", + "required": [ + "script" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "string", + "description": "Description of this context." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A test case.", + "required": [ + "script" + ], + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "context": { + "type": "string", + "description": "Description of this context." + }, + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "testcase": { + "oneOf": [ + { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "type": "integer", + "description": "Expected exit code for the run" + } + ] + }, + "template": { + "oneOf": [ + { + "type": "string", + "description": "Name of the template to insert." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Name of the template to insert." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "parameters": { + "oneOf": [ + { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "repeat": { + "oneOf": [ + { + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": [ + "template", + "parameters" + ], + "properties": { + "template": { + "oneOf": [ + { + "type": "string", + "description": "Name of the template to insert." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Name of the template to insert." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "parameters": { + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": [ + "template", + "parameters" + ], + "properties": { + "template": { + "type": "string", + "description": "Name of the template to insert." + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + } + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "integer", + "description": "Expected exit code for the run" + } + ] + }, + "template": { + "type": "string", + "description": "Name of the template to insert." + }, + "parameters": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "repeat": { + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": [ + "template", + "parameters" + ], + "properties": { + "template": { + "type": "string", + "description": "Name of the template to insert." + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "testcase_without_templates": { + "oneOf": [ + { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "type": "integer", + "description": "Expected exit code for the run" + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { "anyOf": [ { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "The key of the parameter.", + "type": "string" }, { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "arguments": { - "oneOf": [ - { - "description": "Array of program call arguments", - "type": "array", - "items": { - "oneOf": [ - { + "description": "Stdin for this context", "type": [ "string", "number", "integer", "boolean" ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" }, { - "type": "object", - "additionalProperties": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - "not": { + "description": "Array of program call arguments", + "type": "array", + "items": { "anyOf": [ { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "The key of the parameter.", + "type": "string" }, { - "required": [ - "types" + "type": [ + "string", + "number", + "integer", + "boolean" ] } ] } } ] - } - }, - { - "type": "object", - "additionalProperties": { - "description": "Array of program call arguments", - "type": "array", - "items": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "statement": { - "description": "The statement to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "expression": { - "description": "The expression to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "exception": { - "description": "Expected exception message", - "$ref": "#/subDefinitions/exceptionChannel" - }, - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "additionalProperties": { + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { "$ref": "#/subDefinitions/file" } }, - "not": { + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { "anyOf": [ { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "The key of the parameter.", + "type": "string" }, { - "required": [ - "types" - ] + "type": "integer", + "description": "Expected exit code for the run" } ] } } - ] - }, - "return": { - "description": "Expected return value", - "$ref": "#/subDefinitions/returnOutputChannel" - }, - "stderr": { - "description": "Expected output at stderr", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "stdout": { - "description": "Expected output at stdout", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "file": { - "description": "Expected files generated by the submission.", - "$ref": "#/subDefinitions/fileOutputChannel" - }, - "exit_code": { - "type": "integer", - "description": "Expected exit code for the run" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } } - } + ] } }, "subDefinitions": { @@ -1349,6 +2882,10 @@ { "$ref": "#/subDefinitions/yamlValue" }, + { + "type": "string", + "description": "The key of the parameter." + }, { "type": "string", "format": "tested-dsl-expression", @@ -1834,71 +3371,109 @@ ] }, "arguments": { - "oneOf": [ + "anyOf": [ { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] } - ] - } + } + ] }, { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } + } + ] } ] }, @@ -1971,11 +3546,19 @@ "default": "evaluate" }, "arguments": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + ] }, "languages": { "type": "array", @@ -2300,71 +3883,109 @@ ] }, "arguments": { - "oneOf": [ + "anyOf": [ { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] } - ] - } + } + ] }, { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } + } + ] } ] }, @@ -2443,11 +4064,19 @@ "default": "evaluate" }, "arguments": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + ] }, "languages": { "type": "array", @@ -2655,103 +4284,34 @@ "description" ] }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "name": { - "oneOf": [ - { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - }, - { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "arguments": { - "oneOf": [ - { - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] } ] } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, { "type": "object", "additionalProperties": { - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" }, "not": { "anyOf": [ @@ -2775,6 +4335,113 @@ } ] }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, "languages": { "oneOf": [ { @@ -2844,11 +4511,19 @@ "default": "evaluate" }, "arguments": { - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + ] }, "languages": { "type": "array", @@ -3043,130 +4718,8 @@ "name": { "type": "string", "description": "The name of the custom check function.", - "default": "evaluate" - } - } - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "arguments": { - "oneOf": [ - { - "minProperties": 1, - "description": "Language mapping of oracle arguments.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "oneOf": [ - { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "oneOf": [ - { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, - { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - } - }, - { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - } - }, - { - "type": "object", - "additionalProperties": { - "minProperties": 1, - "description": "Language mapping of oracle arguments.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." + "default": "evaluate" + } } } }, @@ -3192,6 +4745,166 @@ } ] }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + }, + { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + ] + }, "value": { "oneOf": [ { @@ -3265,20 +4978,28 @@ } }, "arguments": { - "minProperties": 1, - "description": "Language mapping of oracle arguments.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } } - } + ] }, "value": { "$ref": "#/subDefinitions/yamlValueOrPythonExpression" @@ -3399,163 +5120,429 @@ } ] } - }, + }, + { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "oneOf": [ + { + "description": "The message to display.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The message to display.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "format": { + "oneOf": [ + { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + { + "type": "object", + "additionalProperties": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + } + } + }, + { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "description": "The message to display.", + "type": "string" + }, + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + "textConfigurationOptions": { + "oneOf": [ { "type": "object", - "required": [ - "description" - ], + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, "properties": { - "description": { - "oneOf": [ + "applyRounding": { + "anyOf": [ { - "description": "The message to display.", - "type": "string" + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] }, { - "type": "object", - "additionalProperties": { - "description": "The message to display.", - "type": "string" - }, - "not": { - "anyOf": [ - { - "required": [ - "description" + "description": "Apply rounding when comparing as float", + "type": "boolean" + } + ] + }, + "caseInsensitive": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + { + "description": "Ignore case when comparing strings", + "type": "boolean" + } + ] + }, + "ignoreWhitespace": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" }, - { - "required": [ - "value" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] + } + } + ] + }, + { + "description": "Ignore trailing whitespace", + "type": "boolean" + } + ] + }, + "normalizeTrailingNewlines": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } + } + ] + }, + { + "description": "Normalize trailing newlines", + "type": "boolean" } ] }, - "format": { - "oneOf": [ + "roundTo": { + "anyOf": [ { - "type": "string", - "default": "text", - "description": "The format of the message, either a programming language, 'text' or 'html'." + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] + }, + { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + } + ] + }, + "tryFloatingPoint": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } + ] }, { - "type": "object", - "additionalProperties": { - "type": "string", - "default": "text", - "description": "The format of the message, either a programming language, 'text' or 'html'." - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] + "description": "Try comparing text as floating point numbers", + "type": "boolean" + } + ] + }, + "hideExpected": { + "anyOf": [ + { + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } + } + ] + }, + { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" } ] } } }, - { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "description" - ], - "properties": { - "description": { - "description": "The message to display.", - "type": "string" - }, - "format": { - "type": "string", - "default": "text", - "description": "The format of the message, either a programming language, 'text' or 'html'." - } - } - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" - ] - }, - { - "required": [ - "types" - ] - } - ] - } - } - ] - }, - "textConfigurationOptions": { - "oneOf": [ - { - "type": "object", - "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties": 1, - "properties": { - "applyRounding": { - "description": "Apply rounding when comparing as float", - "type": "boolean" - }, - "caseInsensitive": { - "description": "Ignore case when comparing strings", - "type": "boolean" - }, - "ignoreWhitespace": { - "description": "Ignore trailing whitespace", - "type": "boolean" - }, - "normalizeTrailingNewlines": { - "description": "Normalize trailing newlines", - "type": "boolean" - }, - "roundTo": { - "description": "The number of decimals to round at, when applying the rounding on floats", - "type": "integer" - }, - "tryFloatingPoint": { - "description": "Try comparing text as floating point numbers", - "type": "boolean" - }, - "hideExpected": { - "description": "Hide the expected value in feedback (default: false), not recommended to use!", - "type": "boolean" - } - } - }, { "type": "object", "additionalProperties": { @@ -3564,32 +5551,88 @@ "minProperties": 1, "properties": { "applyRounding": { - "description": "Apply rounding when comparing as float", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Apply rounding when comparing as float", + "type": "boolean" + } + ] }, "caseInsensitive": { - "description": "Ignore case when comparing strings", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Ignore case when comparing strings", + "type": "boolean" + } + ] }, "ignoreWhitespace": { - "description": "Ignore trailing whitespace", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Ignore trailing whitespace", + "type": "boolean" + } + ] }, "normalizeTrailingNewlines": { - "description": "Normalize trailing newlines", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Normalize trailing newlines", + "type": "boolean" + } + ] }, "roundTo": { - "description": "The number of decimals to round at, when applying the rounding on floats", - "type": "integer" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + } + ] }, "tryFloatingPoint": { - "description": "Try comparing text as floating point numbers", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + } + ] }, "hideExpected": { - "description": "Hide the expected value in feedback (default: false), not recommended to use!", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + ] } } }, @@ -3709,46 +5752,84 @@ ] }, "textualType": { - "oneOf": [ + "anyOf": [ { - "description": "Simple textual value, converted to string.", - "type": [ - "string", - "number", - "integer", - "boolean" + "oneOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "object", + "additionalProperties": { + "description": "The key of the parameter.", + "type": "string" + }, + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } + ] + } + } ] }, { - "type": "object", - "additionalProperties": { - "description": "Simple textual value, converted to string.", - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - "not": { - "anyOf": [ - { - "required": [ - "description" - ] - }, - { - "required": [ - "value" + "oneOf": [ + { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "additionalProperties": { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" ] }, - { - "required": [ - "types" + "not": { + "anyOf": [ + { + "required": [ + "description" + ] + }, + { + "required": [ + "value" + ] + }, + { + "required": [ + "types" + ] + } ] } - ] - } + } + ] } ] }, diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json index 65083de8..470a7408 100644 --- a/tested/dsl/schema-strict-nat-translation.json +++ b/tested/dsl/schema-strict-nat-translation.json @@ -15,123 +15,249 @@ ], "definitions": { "_rootObject": { - "type": "object", "oneOf": [ { - "required": [ - "tabs" - ], - "not": { - "required": [ - "units" - ] - } - }, - { - "required": [ - "units" - ], - "not": { - "required": [ - "tabs" - ] - } - } - ], - "properties": { - "files": { + "type": "object", "oneOf": [ { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" + "required": [ + "tabs" + ], + "not": { + "required": [ + "units" + ] } }, { - "type": "object", "required": [ - "__tag__", - "value" + "units" ], - "properties": { - "__tag__": { + "not": { + "required": [ + "tabs" + ] + } + } + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } + ] + }, + "namespace": { + "oneOf": [ + { "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "description": "Namespace of the submitted solution, in `snake_case`" }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Namespace of the submitted solution, in `snake_case`" + } + } + } + } + ] + }, + "tabs": { + "$ref": "#/definitions/_tabList" + }, + "units": { + "$ref": "#/definitions/_unitList" + }, + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ + { + "$ref": "#/subDefinitions/programmingLanguage" + }, + { + "const": "tested" + } + ] + }, + "definitions": { + "oneOf": [ + { + "description": "Define hashes to use elsewhere.", + "type": "object" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Define hashes to use elsewhere.", + "type": "object" + } } } } + ] + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" } } - ] + } }, - "namespace": { - "oneOf": [ - { + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "Namespace of the submitted solution, in `snake_case`" + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { + "additionalProperties": { + "type": "object", + "oneOf": [ + { + "required": [ + "tabs" + ], + "not": { + "required": [ + "units" + ] + } + }, + { + "required": [ + "units" + ], + "not": { + "required": [ + "tabs" + ] + } + } + ], + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "namespace": { "type": "string", "description": "Namespace of the submitted solution, in `snake_case`" + }, + "tabs": { + "$ref": "#/definitions/_tabList" + }, + "units": { + "$ref": "#/definitions/_unitList" + }, + "language": { + "description": "Indicate that all code is in a specific language.", + "oneOf": [ + { + "$ref": "#/subDefinitions/programmingLanguage" + }, + { + "const": "tested" + } + ] + }, + "definitions": { + "description": "Define hashes to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } } } } } - ] - }, - "tabs": { - "$ref": "#/definitions/_tabList" - }, - "units": { - "$ref": "#/definitions/_unitList" - }, - "language": { - "description": "Indicate that all code is in a specific language.", - "oneOf": [ - { - "$ref": "#/subDefinitions/programmingLanguage" - }, - { - "const": "tested" - } - ] - }, - "definitions": { - "description": "Define hashes to use elsewhere.", - "type": "object" - }, - "config": { - "$ref": "#/subDefinitions/inheritableConfigObject" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." + } } - } + ] }, "_tabList": { "oneOf": [ @@ -204,234 +330,210 @@ ] }, "tab": { - "type": "object", - "description": "A tab in the test suite.", - "required": [ - "tab" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "oneOf": [ + { + "type": "object", + "description": "A tab in the test suite.", + "required": [ + "tab" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - } - } - } - } - ] - }, - "hidden": { - "type": "boolean", - "description": "Defines if the unit/tab is hidden for the student or not" - }, - "tab": { - "oneOf": [ - { - "description": "The name of this tab.", - "type": "string" - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } + ] + }, + "hidden": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] }, - "value": { + { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + } + ] + }, + "tab": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { "type": "object", - "additionalProperties": { - "description": "The name of this tab.", - "type": "string" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" + } + } } } - } - } - ] - }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" - }, - "config": { - "$ref": "#/subDefinitions/inheritableConfigObject" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." - } - }, - "oneOf": [ - { - "required": [ - "contexts" - ], - "properties": { - "contexts": { - "$ref": "#/definitions/_contextList" - } - } - }, - { - "required": [ - "testcases" - ], - "properties": { - "testcases": { - "$ref": "#/definitions/_testcaseList" - } - } - } - ] - }, - "unit": { - "type": "object", - "description": "A unit in the test suite.", - "required": [ - "unit" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } + ] }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "definitions": { + "oneOf": [ + { + "description": "Define objects to use elsewhere.", + "type": "object" }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Define objects to use elsewhere.", + "type": "object" + } } } } + ] + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" } } - ] - }, - "hidden": { - "type": "boolean", - "description": "Defines if the unit/tab is hidden for the student or not" - }, - "unit": { + }, "oneOf": [ { - "description": "The name of this tab.", - "type": "string" + "required": [ + "contexts" + ], + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" + } + } }, { - "type": "object", "required": [ - "__tag__", - "value" + "testcases" ], "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "description": "The name of this tab.", - "type": "string" - } + "testcases": { + "$ref": "#/definitions/_testcaseList" } } } ] }, - "definitions": { - "description": "Define objects to use elsewhere.", - "type": "object" - }, - "config": { - "$ref": "#/subDefinitions/inheritableConfigObject" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." - } - }, - "oneOf": [ - { - "required": [ - "cases" - ], - "properties": { - "cases": { - "$ref": "#/definitions/_caseList" - } - } - }, - { - "required": [ - "scripts" - ], - "properties": { - "scripts": { - "$ref": "#/definitions/_scriptList" - } - } - } - ] - }, - "_contextList": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/context" - } - }, { "type": "object", "required": [ @@ -447,95 +549,286 @@ "value": { "type": "object", "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/context" - } - } - } - } - } - ] - }, - "_caseList": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/case" - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/case" - } + "type": "object", + "description": "A tab in the test suite.", + "required": [ + "tab" + ], + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "hidden": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + } + ] + }, + "tab": { + "description": "The name of this tab.", + "type": "string" + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + }, + "oneOf": [ + { + "required": [ + "contexts" + ], + "properties": { + "contexts": { + "$ref": "#/definitions/_contextList" + } + } + }, + { + "required": [ + "testcases" + ], + "properties": { + "testcases": { + "$ref": "#/definitions/_testcaseList" + } + } + } + ] } } } } ] }, - "_testcaseList": { + "unit": { "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/testcase" - } - }, { "type": "object", + "description": "A unit in the test suite.", "required": [ - "__tag__", - "value" + "unit" ], "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } + ] }, - "value": { + "hidden": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + } + ] + }, + "unit": { + "oneOf": [ + { + "description": "The name of this tab.", + "type": "string" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "The name of this tab.", + "type": "string" + } + } + } + } + ] + }, + "definitions": { + "oneOf": [ + { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Define objects to use elsewhere.", + "type": "object" + } + } + } + } + ] + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/testcase" + "$ref": "#/definitions/testcase_without_templates" + } + } + }, + "oneOf": [ + { + "required": [ + "cases" + ], + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" + } + } + }, + { + "required": [ + "scripts" + ], + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" } } } - } - } - ] - }, - "_scriptList": { - "oneOf": [ - { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/script" - } + ] }, { "type": "object", @@ -552,564 +845,452 @@ "value": { "type": "object", "additionalProperties": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/script" - } - } - } - } - } - ] - }, - "context": { - "type": "object", - "description": "A set of testcase in the same context.", - "required": [ - "testcases" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { + "type": "object", + "description": "A unit in the test suite.", + "required": [ + "unit" + ], + "properties": { + "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { "$ref": "#/subDefinitions/file" } + }, + "hidden": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "boolean", + "description": "Defines if the unit/tab is hidden for the student or not" + } + ] + }, + "unit": { + "description": "The name of this tab.", + "type": "string" + }, + "definitions": { + "description": "Define objects to use elsewhere.", + "type": "object" + }, + "config": { + "$ref": "#/subDefinitions/inheritableConfigObject" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } } - } + }, + "oneOf": [ + { + "required": [ + "cases" + ], + "properties": { + "cases": { + "$ref": "#/definitions/_caseList" + } + } + }, + { + "required": [ + "scripts" + ], + "properties": { + "scripts": { + "$ref": "#/definitions/_scriptList" + } + } + } + ] } } - ] + } + } + ] + }, + "_contextList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" + } }, - "context": { - "oneOf": [ - { + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "Description of this context." + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Description of this context." - } + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/context" } } } - ] - }, - "testcases": { - "$ref": "#/definitions/_testcaseList" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." + } } - } + ] }, - "case": { - "type": "object", - "description": "A test case.", - "required": [ - "script" - ], - "properties": { - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } + "_caseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - } + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/case" } } } - ] + } + } + ] + }, + "_testcaseList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } }, - "context": { - "oneOf": [ - { + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "Description of this context." + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Description of this context." - } + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } } } - ] - }, - "script": { - "$ref": "#/definitions/_scriptList" - }, - "translations": { - "type": "object", - "description": "Define translations in the global scope." + } } - } + ] }, - "testcase": { - "type": "object", - "description": "An individual test for a statement or expression", - "additionalProperties": false, - "properties": { - "description": { - "$ref": "#/subDefinitions/message" + "_scriptList": { + "oneOf": [ + { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" + } }, - "stdin": { - "oneOf": [ - { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" - ] + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/definitions/testcase" } } } - ] - }, - "arguments": { - "oneOf": [ - { - "description": "Array of program call arguments", - "type": "array", - "items": { - "oneOf": [ - { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] + } + } + ] + }, + "context": { + "oneOf": [ + { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ + "testcases" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } } } } - ] - } + } + ] }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { + "context": { + "oneOf": [ + { "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "description": "Description of this context." }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "Array of program call arguments", - "type": "array", - "items": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Description of this context." + } } } } + ] + }, + "testcases": { + "$ref": "#/definitions/_testcaseList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" } } - ] - }, - "statement": { - "description": "The statement to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "expression": { - "description": "The expression to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "exception": { - "description": "Expected exception message", - "$ref": "#/subDefinitions/exceptionChannel" + } }, - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { + "additionalProperties": { + "type": "object", + "description": "A set of testcase in the same context.", + "required": [ + "testcases" + ], + "properties": { + "files": { "description": "A list of files used in the test suite.", "type": "array", "items": { "$ref": "#/subDefinitions/file" } + }, + "context": { + "type": "string", + "description": "Description of this context." + }, + "testcases": { + "$ref": "#/definitions/_testcaseList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } } } } } - ] - }, - "return": { - "description": "Expected return value", - "$ref": "#/subDefinitions/returnOutputChannel" - }, - "stderr": { - "description": "Expected output at stderr", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "stdout": { - "description": "Expected output at stdout", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "file": { - "description": "Expected files generated by the submission.", - "$ref": "#/subDefinitions/fileOutputChannel" - }, - "exit_code": { - "type": "integer", - "description": "Expected exit code for the run" + } } - } + ] }, - "script": { - "type": "object", - "description": "An individual test (script) for a statement or expression", - "properties": { - "description": { - "$ref": "#/subDefinitions/message" - }, - "stdin": { - "oneOf": [ - { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" + "case": { + "oneOf": [ + { + "type": "object", + "description": "A test case.", + "required": [ + "script" + ], + "properties": { + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } ] }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { + "context": { + "oneOf": [ + { "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "description": "Description of this context." }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } - } - } - } - ] - }, - "arguments": { - "oneOf": [ - { - "description": "Array of program call arguments", - "type": "array", - "items": { - "oneOf": [ - { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } + "description": "Description of this context." } } } - ] - } + } + ] }, - { + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "description": "Array of program call arguments", - "type": "array", - "items": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } - } - } - } - } - ] - }, - "statement": { - "description": "The statement to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "expression": { - "description": "The expression to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "exception": { - "description": "Expected exception message", - "$ref": "#/subDefinitions/exceptionChannel" - }, - "files": { - "oneOf": [ - { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } + "description": "Define translations." }, - { + "templates": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - } - } + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" } } - ] - }, - "return": { - "description": "Expected return value", - "$ref": "#/subDefinitions/returnOutputChannel" - }, - "stderr": { - "description": "Expected output at stderr", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "stdout": { - "description": "Expected output at stdout", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "file": { - "description": "Expected files generated by the submission.", - "$ref": "#/subDefinitions/fileOutputChannel" - }, - "exit_code": { - "type": "integer", - "description": "Expected exit code for the run" - } - } - } - }, - "subDefinitions": { - "expressionOrStatement": { - "oneOf": [ - { - "type": "string", - "format": "tested-dsl-expression", - "description": "A statement of expression in Python-like syntax as YAML string." + } }, { "type": "object", @@ -1126,87 +1307,379 @@ "value": { "type": "object", "additionalProperties": { - "type": "string", - "format": "tested-dsl-expression", - "description": "A statement of expression in Python-like syntax as YAML string." + "type": "object", + "description": "A test case.", + "required": [ + "script" + ], + "properties": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "context": { + "type": "string", + "description": "Description of this context." + }, + "script": { + "$ref": "#/definitions/_scriptList" + }, + "translations": { + "type": "object", + "description": "Define translations." + }, + "templates": { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates" + } + } + } } } } - }, + } + ] + }, + "testcase": { + "oneOf": [ { - "anyOf": [ - { - "oneOf": [ + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "anyOf": [ { - "type": "object", - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "oneOf": [ - { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { "type": "object", - "additionalProperties": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } } } } } - ] - } + } + ] }, { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] }, - "value": { + { "type": "object", - "additionalProperties": { - "type": "object", - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "items": { + "value": { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "A language-specific literal, which will be used verbatim." + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } } } } - } + ] + }, + { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + ] + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + } + } + } + } + } + ] } ] }, - { + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, { "type": "object", "required": [ @@ -1217,21 +1690,75 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!programming_language" + "const": "!natural_language" }, "value": { - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "oneOf": [ - { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, - { + "type": "object", + "additionalProperties": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } + } + } + ] + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { "type": "object", "required": [ "__tag__", @@ -1241,22 +1768,30 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!natural_language" + "const": "!parameter" }, "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - } + "description": "The key of the parameter.", + "type": "string" } } } - ] - }, - "type": "object" + } + } } - } + ] + }, + { + "type": "integer", + "description": "Expected exit code for the run" + } + ] + }, + "template": { + "oneOf": [ + { + "type": "string", + "description": "Name of the template to insert." }, { "type": "object", @@ -1273,6 +1808,25 @@ "value": { "type": "object", "additionalProperties": { + "type": "string", + "description": "Name of the template to insert." + } + } + } + } + ] + }, + "parameters": { + "oneOf": [ + { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { "type": "object", "required": [ "__tag__", @@ -1282,72 +1836,18 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!programming_language" + "const": "!natural_language" }, "value": { - "description": "Programming-language-specific statement or expression.", - "minProperties": 1, - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, - "type": "object" + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } } } } - } + ] } - } - ] - } - ] - } - ] - }, - "yamlValueOrPythonExpression": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValue" - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!expression" - }, - "value": { - "type": "string", - "format": "tested-dsl-expression", - "description": "An expression in Python-syntax." - } - } - } - ] - }, - "file": { - "oneOf": [ - { - "type": "object", - "description": "A file used in the test suite.", - "required": [ - "name", - "url" - ], - "properties": { - "name": { - "oneOf": [ - { - "type": "string", - "description": "The filename, including the file extension." }, { "type": "object", @@ -1364,20 +1864,150 @@ "value": { "type": "object", "additionalProperties": { - "type": "string", - "description": "The filename, including the file extension." + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } } } } } ] }, - "url": { + "repeat": { "oneOf": [ { - "type": "string", - "format": "uri", - "description": "Relative path to the file in the `description` folder of an exercise." + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": [ + "template", + "parameters" + ], + "properties": { + "template": { + "oneOf": [ + { + "type": "string", + "description": "Name of the template to insert." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Name of the template to insert." + } + } + } + } + ] + }, + "parameters": { + "oneOf": [ + { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + } + ] + } + } }, { "type": "object", @@ -1394,9 +2024,28 @@ "value": { "type": "object", "additionalProperties": { - "type": "string", - "format": "uri", - "description": "Relative path to the file in the `description` folder of an exercise." + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": [ + "template", + "parameters" + ], + "properties": { + "template": { + "type": "string", + "description": "Name of the template to insert." + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } } } } @@ -1421,20 +2070,138 @@ "type": "object", "additionalProperties": { "type": "object", - "description": "A file used in the test suite.", - "required": [ - "name", - "url" - ], + "description": "An individual test for a statement or expression", + "additionalProperties": false, "properties": { - "name": { - "type": "string", - "description": "The filename, including the file extension." + "description": { + "$ref": "#/subDefinitions/message" }, - "url": { + "stdin": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + } + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "integer", + "description": "Expected exit code for the run" + } + ] + }, + "template": { "type": "string", - "format": "uri", - "description": "Relative path to the file in the `description` folder of an exercise." + "description": "Name of the template to insert." + }, + "parameters": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "repeat": { + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": [ + "template", + "parameters" + ], + "properties": { + "template": { + "type": "string", + "description": "Name of the template to insert." + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } } } } @@ -1443,103 +2210,339 @@ } ] }, - "exceptionChannel": { + "testcase_without_templates": { "oneOf": [ - { - "type": "string", - "description": "Message of the expected exception." - }, { "type": "object", - "required": [ - "__tag__", - "value" - ], + "description": "An individual test for a statement or expression", + "additionalProperties": false, "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "description": { + "$ref": "#/subDefinitions/message" }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Message of the expected exception." - } - } - } - }, - { - "type": "object", - "required": [ - "types" - ], - "properties": { - "message": { - "oneOf": [ + "stdin": { + "anyOf": [ { - "description": "Message of the expected exception.", - "type": "string" + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] }, { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "oneOf": [ + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "Message of the expected exception.", - "type": "string" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } } } - } + ] } ] }, - "types": { - "oneOf": [ + "arguments": { + "anyOf": [ { - "minProperties": 1, - "description": "Language mapping of expected exception types.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { "type": "object", - "additionalProperties": { - "type": "string" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } } } } } - ] + } + ] + }, + { + "oneOf": [ + { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + } + ] + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Array of program call arguments", + "type": "array", + "items": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + } + } + } + } + } + ] + } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "oneOf": [ + { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } }, { @@ -1557,150 +2560,37 @@ "value": { "type": "object", "additionalProperties": { - "minProperties": 1, - "description": "Language mapping of expected exception types.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, + "description": "A list of files used in the test suite.", + "type": "array", "items": { - "type": "string" + "$ref": "#/subDefinitions/file" } } } } } ] - } - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "types" - ], - "properties": { - "message": { - "description": "Message of the expected exception.", - "type": "string" - }, - "types": { - "minProperties": 1, - "description": "Language mapping of expected exception types.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "type": "string" - } - } - } - } - } - } - } - ] - }, - "textOutputChannel": { - "anyOf": [ - { - "$ref": "#/subDefinitions/textualType" - }, - { - "oneOf": [ - { - "type": "object", - "description": "Built-in oracle for text values.", - "required": [ - "data" - ], - "properties": { - "data": { - "$ref": "#/subDefinitions/textualType" - }, - "oracle": { - "const": "builtin" - }, - "config": { - "$ref": "#/subDefinitions/textConfigurationOptions" - } - } + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "description": "Built-in oracle for text values.", - "required": [ - "data" - ], - "properties": { - "data": { - "$ref": "#/subDefinitions/textualType" - }, - "oracle": { - "const": "builtin" - }, - "config": { - "$ref": "#/subDefinitions/textConfigurationOptions" - } - } - } - } - } - } - ] - }, - { - "oneOf": [ - { - "type": "object", - "description": "Custom oracle for text values.", - "required": [ - "oracle", - "file", - "data" - ], - "properties": { - "data": { - "$ref": "#/subDefinitions/textualType" - }, - "oracle": { - "const": "custom_check" - }, - "file": { + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { "oneOf": [ - { - "type": "string", - "description": "The path to the file containing the custom check function." - }, { "type": "object", "required": [ @@ -1711,25 +2601,13 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!natural_language" + "const": "!parameter" }, "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "The path to the file containing the custom check function." - } + "description": "The key of the parameter.", + "type": "string" } } - } - ] - }, - "name": { - "oneOf": [ - { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" }, { "type": "object", @@ -1746,26 +2624,6 @@ "value": { "type": "object", "additionalProperties": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - } - } - } - } - ] - }, - "arguments": { - "oneOf": [ - { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - { "type": "object", "required": [ "__tag__", @@ -1775,406 +2633,276 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!natural_language" + "const": "!parameter" }, "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "description": "The key of the parameter.", + "type": "string" } } } - ] - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } - } } } } ] }, - "languages": { - "oneOf": [ - { - "type": "array", - "description": "Which programming languages are supported by this oracle.", - "items": { - "$ref": "#/subDefinitions/programmingLanguage" - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "Which programming languages are supported by this oracle.", - "items": { - "$ref": "#/subDefinitions/programmingLanguage" - } - } - } - } - } - ] + { + "type": "integer", + "description": "Expected exit code for the run" } - } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "description": "Custom oracle for text values.", - "required": [ - "oracle", - "file", - "data" - ], - "properties": { - "data": { - "$ref": "#/subDefinitions/textualType" - }, - "oracle": { - "const": "custom_check" - }, - "file": { - "type": "string", - "description": "The path to the file containing the custom check function." - }, - "name": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" + "additionalProperties": { + "type": "object", + "description": "An individual test for a statement or expression", + "additionalProperties": false, + "properties": { + "description": { + "$ref": "#/subDefinitions/message" + }, + "stdin": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" }, - "arguments": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + { + "description": "Stdin for this context", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" }, - "languages": { + { + "description": "Array of program call arguments", "type": "array", - "description": "Which programming languages are supported by this oracle.", "items": { - "$ref": "#/subDefinitions/programmingLanguage" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + ] } } + ] + }, + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement" + }, + "exception": { + "description": "Expected exception message", + "$ref": "#/subDefinitions/exceptionChannel" + }, + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" } + }, + "return": { + "description": "Expected return value", + "$ref": "#/subDefinitions/returnOutputChannel" + }, + "stderr": { + "description": "Expected output at stderr", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "stdout": { + "description": "Expected output at stdout", + "$ref": "#/subDefinitions/textOutputChannel" + }, + "file": { + "description": "Expected files generated by the submission.", + "$ref": "#/subDefinitions/fileOutputChannel" + }, + "exit_code": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "integer", + "description": "Expected exit code for the run" + } + ] } } } } - ] + } } ] - }, - "fileOutputChannel": { - "anyOf": [ + } + }, + "subDefinitions": { + "expressionOrStatement": { + "oneOf": [ { - "oneOf": [ - { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { "type": "object", - "description": "Built-in oracle for files.", - "required": [ - "content", - "location" - ], - "properties": { - "content": { - "oneOf": [ - { - "type": "string", - "description": "Path to the file containing the expected contents, relative to the evaluation directory." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { + "additionalProperties": { + "type": "string", + "format": "tested-dsl-expression", + "description": "A statement of expression in Python-like syntax as YAML string." + } + } + } + }, + { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "Path to the file containing the expected contents, relative to the evaluation directory." + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } } } } - } - ] + ] + } }, - "location": { - "oneOf": [ - { + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "Path to where the file generated by the submission should go." + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "additionalProperties": { + "type": "object", + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Path to where the file generated by the submission should go." - } + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } } - ] - }, - "oracle": { - "const": "builtin" - }, - "config": { - "$ref": "#/subDefinitions/fileConfigurationOptions" + } } - } + ] }, { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { + "oneOf": [ + { "type": "object", - "additionalProperties": { - "type": "object", - "description": "Built-in oracle for files.", - "required": [ - "content", - "location" - ], - "properties": { - "content": { - "type": "string", - "description": "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "location": { - "type": "string", - "description": "Path to where the file generated by the submission should go." - }, - "oracle": { - "const": "builtin" - }, - "config": { - "$ref": "#/subDefinitions/fileConfigurationOptions" - } - } - } - } - } - } - ] - }, - { - "oneOf": [ - { - "type": "object", - "description": "Custom oracle for file values.", - "required": [ - "oracle", - "content", - "location", - "file" - ], - "properties": { - "oracle": { - "const": "custom_check" - }, - "content": { - "oneOf": [ - { - "type": "string", - "description": "Path to the file containing the expected contents, relative to the evaluation directory." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Path to the file containing the expected contents, relative to the evaluation directory." - } - } - } - } - ] - }, - "location": { - "oneOf": [ - { - "type": "string", - "description": "Path to where the file generated by the submission should go." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "Path to where the file generated by the submission should go." - } - } - } - } - ] - }, - "file": { - "oneOf": [ - { - "type": "string", - "description": "The path to the file containing the custom check function." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "The path to the file containing the custom check function." - } - } - } - } - ] - }, - "name": { - "oneOf": [ - { + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" + "description": "The tag used in the yaml", + "const": "!programming_language" }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - } - } - } - } - ] - }, - "arguments": { - "oneOf": [ - { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, "items": { "oneOf": [ { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "type": "string", + "description": "A language-specific literal, which will be used verbatim." }, { "type": "object", @@ -2191,148 +2919,71 @@ "value": { "type": "object", "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "type": "string", + "description": "A language-specific literal, which will be used verbatim." } } } } ] - } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!programming_language" + }, + "value": { + "description": "Programming-language-specific statement or expression.", + "minProperties": 1, + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + "type": "object" } } } } - ] - }, - "languages": { - "oneOf": [ - { - "type": "array", - "description": "Which programming languages are supported by this oracle.", - "items": { - "$ref": "#/subDefinitions/programmingLanguage" - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "Which programming languages are supported by this oracle.", - "items": { - "$ref": "#/subDefinitions/programmingLanguage" - } - } - } - } - } - ] - } - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "description": "Custom oracle for file values.", - "required": [ - "oracle", - "content", - "location", - "file" - ], - "properties": { - "oracle": { - "const": "custom_check" - }, - "content": { - "type": "string", - "description": "Path to the file containing the expected contents, relative to the evaluation directory." - }, - "location": { - "type": "string", - "description": "Path to where the file generated by the submission should go." - }, - "file": { - "type": "string", - "description": "The path to the file containing the custom check function." - }, - "name": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - }, - "arguments": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } - }, - "languages": { - "type": "array", - "description": "Which programming languages are supported by this oracle.", - "items": { - "$ref": "#/subDefinitions/programmingLanguage" - } - } - } } } - } + ] } ] } ] }, - "returnOutputChannel": { + "yamlValueOrPythonExpression": { "oneOf": [ { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "$ref": "#/subDefinitions/yamlValue" }, { "type": "object", @@ -2344,13 +2995,11 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!natural_language" + "const": "!parameter" }, "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "type": "string", + "description": "The key of the parameter." } } }, @@ -2364,46 +3013,86 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!oracle" + "const": "!expression" }, "value": { - "additionalProperties": false, - "required": [ - "value" - ], - "properties": { - "oracle": { - "const": "builtin" + "type": "string", + "format": "tested-dsl-expression", + "description": "An expression in Python-syntax." + } + } + } + ] + }, + "file": { + "oneOf": [ + { + "type": "object", + "description": "A file used in the test suite.", + "required": [ + "name", + "url" + ], + "properties": { + "name": { + "oneOf": [ + { + "type": "string", + "description": "The filename, including the file extension." }, - "value": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - { + "value": { "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } - } + "additionalProperties": { + "type": "string", + "description": "The filename, including the file extension." } } - ] + } } - }, - "type": "object" + ] + }, + "url": { + "oneOf": [ + { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." + } + } + } + } + ] } } }, @@ -2423,35 +3112,33 @@ "type": "object", "additionalProperties": { "type": "object", + "description": "A file used in the test suite.", "required": [ - "__tag__", - "value" + "name", + "url" ], "properties": { - "__tag__": { + "name": { "type": "string", - "description": "The tag used in the yaml", - "const": "!oracle" + "description": "The filename, including the file extension." }, - "value": { - "additionalProperties": false, - "required": [ - "value" - ], - "properties": { - "oracle": { - "const": "builtin" - }, - "value": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } - }, - "type": "object" + "url": { + "type": "string", + "format": "uri", + "description": "Relative path to the file in the `description` folder of an exercise." } } } } } + } + ] + }, + "exceptionChannel": { + "oneOf": [ + { + "type": "string", + "description": "Message of the expected exception." }, { "type": "object", @@ -2463,45 +3150,241 @@ "__tag__": { "type": "string", "description": "The tag used in the yaml", - "const": "!oracle" + "const": "!natural_language" }, "value": { - "additionalProperties": false, - "required": [ - "value", - "oracle", - "file" - ], - "properties": { - "oracle": { - "const": "custom_check" - }, - "value": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Message of the expected exception." + } + } + } + }, + { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "oneOf": [ + { + "description": "Message of the expected exception.", + "type": "string" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Message of the expected exception.", + "type": "string" + } + } + } + } + ] + }, + "types": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string" + } } } } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } } - ] + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "types" + ], + "properties": { + "message": { + "description": "Message of the expected exception.", + "type": "string" + }, + "types": { + "minProperties": 1, + "description": "Language mapping of expected exception types.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "string" + } + } + } + } + } + } + } + ] + }, + "textOutputChannel": { + "anyOf": [ + { + "$ref": "#/subDefinitions/textualType" + }, + { + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for text values.", + "required": [ + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/textConfigurationOptions" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for text values.", + "required": [ + "oracle", + "file", + "data" + ], + "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, + "oracle": { + "const": "custom_check" }, "file": { "oneOf": [ @@ -2564,61 +3447,122 @@ ] }, "arguments": { - "oneOf": [ + "anyOf": [ { - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "type": "array", - "items": { - "oneOf": [ - { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } } } } } - ] - } + } + ] }, { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } } } } - } + ] } ] }, @@ -2657,51 +3601,37 @@ } ] } - }, - "type": "object" - } - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + } }, - "value": { + { "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!oracle" - }, - "value": { - "additionalProperties": false, + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for text values.", "required": [ - "value", "oracle", - "file" + "file", + "data" ], "properties": { + "data": { + "$ref": "#/subDefinitions/textualType" + }, "oracle": { "const": "custom_check" }, - "value": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - }, "file": { "type": "string", "description": "The path to the file containing the custom check function." @@ -2712,11 +3642,19 @@ "default": "evaluate" }, "arguments": { - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" - } + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + ] }, "languages": { "type": "array", @@ -2725,153 +3663,63 @@ "$ref": "#/subDefinitions/programmingLanguage" } } - }, - "type": "object" + } } } } } - } - }, + ] + } + ] + }, + "fileOutputChannel": { + "anyOf": [ { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!oracle" - }, - "value": { - "additionalProperties": false, + "oneOf": [ + { + "type": "object", + "description": "Built-in oracle for files.", "required": [ - "oracle", - "functions" + "content", + "location" ], "properties": { - "oracle": { - "const": "specific_check" + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + } + } + } + } + ] }, - "functions": { + "location": { "oneOf": [ { - "minProperties": 1, - "description": "Language mapping of oracle functions.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "oneOf": [ - { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "oneOf": [ - { - "type": "string", - "description": "The path to the file containing the custom check function." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "The path to the file containing the custom check function." - } - } - } - } - ] - }, - "name": { - "oneOf": [ - { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - } - } - } - } - ] - } - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string", - "description": "The path to the file containing the custom check function." - }, - "name": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - } - } - } - } - } - } - ] - } - }, - { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + { "type": "object", "required": [ "__tag__", @@ -2886,106 +3734,85 @@ "value": { "type": "object", "additionalProperties": { - "minProperties": 1, - "description": "Language mapping of oracle functions.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "type": "object", - "required": [ - "file" - ], - "properties": { - "file": { - "type": "string", - "description": "The path to the file containing the custom check function." - }, - "name": { - "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" - } - } - } + "type": "string", + "description": "Path to where the file generated by the submission should go." } } } } ] }, - "arguments": { - "oneOf": [ - { - "minProperties": 1, - "description": "Language mapping of oracle arguments.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Built-in oracle for files.", + "required": [ + "content", + "location" + ], + "properties": { + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, - "items": { - "oneOf": [ - { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "oneOf": [ - { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - } - } - } - } - ] - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - } - } - } - } - } - ] + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "oracle": { + "const": "builtin" + }, + "config": { + "$ref": "#/subDefinitions/fileConfigurationOptions" } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "oneOf": [ + { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." }, { "type": "object", @@ -3002,30 +3829,19 @@ "value": { "type": "object", "additionalProperties": { - "minProperties": 1, - "description": "Language mapping of oracle arguments.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { - "type": "array", - "description": "List of YAML (or tagged expression) values to use as arguments to the function.", - "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." - } - } + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." } } } } ] }, - "value": { + "location": { "oneOf": [ { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "type": "string", + "description": "Path to where the file generated by the submission should go." }, { "type": "object", @@ -3042,168 +3858,310 @@ "value": { "type": "object", "additionalProperties": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + "type": "string", + "description": "Path to where the file generated by the submission should go." } } } } ] - } - }, - "type": "object" - } - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!oracle" - }, - "value": { - "additionalProperties": false, - "required": [ - "oracle", - "functions" - ], - "properties": { - "oracle": { - "const": "specific_check" - }, - "functions": { - "minProperties": 1, - "description": "Language mapping of oracle functions.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" + }, + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" }, - "items": { + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { "type": "object", "required": [ - "file" + "__tag__", + "value" ], "properties": { - "file": { + "__tag__": { "type": "string", - "description": "The path to the file containing the custom check function." + "description": "The tag used in the yaml", + "const": "!parameter" }, - "name": { + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { "type": "string", - "description": "The name of the custom check function.", - "default": "evaluate" + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } } } } - }, - "arguments": { - "minProperties": 1, - "description": "Language mapping of oracle arguments.", - "type": "object", - "propertyNames": { - "$ref": "#/subDefinitions/programmingLanguage" - }, - "items": { + ] + }, + { + "oneOf": [ + { "type": "array", "description": "List of YAML (or tagged expression) values to use as arguments to the function.", "items": { - "type": "string", - "description": "A language-specific literal, which will be used verbatim." + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } } } - }, - "value": { - "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + ] + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" } }, - "type": "object" - } + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + ] } } - } - } - } - ] - }, - "programmingLanguage": { - "oneOf": [ - { - "type": "string", - "description": "One of the programming languages supported by TESTed.", - "enum": [ - "bash", - "c", - "haskell", - "java", - "javascript", - "typescript", - "kotlin", - "python", - "runhaskell", - "csharp", - "cpp" - ] - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" }, - "value": { + { "type": "object", - "additionalProperties": { - "type": "string", - "description": "One of the programming languages supported by TESTed.", - "enum": [ - "bash", - "c", - "haskell", - "java", - "javascript", - "typescript", - "kotlin", - "python", - "runhaskell", - "csharp", - "cpp" - ] - } - } - } - } - ] - }, - "message": { + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Custom oracle for file values.", + "required": [ + "oracle", + "content", + "location", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "content": { + "type": "string", + "description": "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location": { + "type": "string", + "description": "Path to where the file generated by the submission should go." + }, + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + ] + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + } + } + ] + } + ] + }, + "returnOutputChannel": { "oneOf": [ { - "type": "string", - "description": "A simple message to display." + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, { "type": "object", @@ -3220,8 +4178,7 @@ "value": { "type": "object", "additionalProperties": { - "type": "string", - "description": "A simple message to display." + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" } } } @@ -3229,141 +4186,1578 @@ { "type": "object", "required": [ - "description" + "__tag__", + "value" ], "properties": { - "description": { - "oneOf": [ - { - "description": "The message to display.", - "type": "string" + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "The message to display.", - "type": "string" + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } } } + ] + } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value" + ], + "properties": { + "oracle": { + "const": "builtin" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" } } - ] + } + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" }, - "format": { - "oneOf": [ - { - "type": "string", - "default": "text", - "description": "The format of the message, either a programming language, 'text' or 'html'." + "value": { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + }, + "file": { + "oneOf": [ + { "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "description": "The path to the file containing the custom check function." }, - "value": { + { "type": "object", - "additionalProperties": { + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + } + ] + } + ] + }, + "languages": { + "oneOf": [ + { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + } + } + } + ] + } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "value", + "oracle", + "file" + ], + "properties": { + "oracle": { + "const": "custom_check" + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + "file": { "type": "string", - "default": "text", - "description": "The format of the message, either a programming language, 'text' or 'html'." + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + ] + }, + "languages": { + "type": "array", + "description": "Which programming languages are supported by this oracle.", + "items": { + "$ref": "#/subDefinitions/programmingLanguage" + } + } + }, + "type": "object" + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "oneOf": [ + { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The path to the file containing the custom check function." + } + } + } + } + ] + }, + "name": { + "oneOf": [ + { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + } + } + } + } + ] + }, + "arguments": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "oneOf": [ + { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "oneOf": [ + { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "oneOf": [ + { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + } + ] + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + } + } + } + ] + } + ] + }, + "value": { + "oneOf": [ + { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + } + } + } + ] + } + }, + "type": "object" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!oracle" + }, + "value": { + "additionalProperties": false, + "required": [ + "oracle", + "functions" + ], + "properties": { + "oracle": { + "const": "specific_check" + }, + "functions": { + "minProperties": 1, + "description": "Language mapping of oracle functions.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "object", + "required": [ + "file" + ], + "properties": { + "file": { + "type": "string", + "description": "The path to the file containing the custom check function." + }, + "name": { + "type": "string", + "description": "The name of the custom check function.", + "default": "evaluate" + } + } + } + }, + "arguments": { + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "minProperties": 1, + "description": "Language mapping of oracle arguments.", + "type": "object", + "propertyNames": { + "$ref": "#/subDefinitions/programmingLanguage" + }, + "items": { + "type": "array", + "description": "List of YAML (or tagged expression) values to use as arguments to the function.", + "items": { + "type": "string", + "description": "A language-specific literal, which will be used verbatim." + } + } + } + ] + }, + "value": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + } + }, + "type": "object" + } + } + } + } + } + } + ] + }, + "programmingLanguage": { + "oneOf": [ + { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "One of the programming languages supported by TESTed.", + "enum": [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp", + "cpp" + ] + } + } + } + } + ] + }, + "message": { + "oneOf": [ + { + "type": "string", + "description": "A simple message to display." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "description": "A simple message to display." + } + } + } + }, + { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "oneOf": [ + { + "description": "The message to display.", + "type": "string" + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "The message to display.", + "type": "string" + } + } + } + } + ] + }, + "format": { + "oneOf": [ + { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "description" + ], + "properties": { + "description": { + "description": "The message to display.", + "type": "string" + }, + "format": { + "type": "string", + "default": "text", + "description": "The format of the message, either a programming language, 'text' or 'html'." + } + } + } + } + } + } + ] + }, + "textConfigurationOptions": { + "oneOf": [ + { + "type": "object", + "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties": 1, + "properties": { + "applyRounding": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "description": "Apply rounding when comparing as float", + "type": "boolean" + } + ] + }, + "caseInsensitive": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "description": "Ignore case when comparing strings", + "type": "boolean" + } + ] + }, + "ignoreWhitespace": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "description": "Ignore trailing whitespace", + "type": "boolean" + } + ] + }, + "normalizeTrailingNewlines": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "description": "Normalize trailing newlines", + "type": "boolean" + } + ] + }, + "roundTo": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + } + ] + }, + "tryFloatingPoint": { + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } } } - } + ] + }, + { + "description": "Try comparing text as floating point numbers", + "type": "boolean" } ] - } - } - }, - { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" - }, - "value": { - "type": "object", - "additionalProperties": { - "type": "object", - "required": [ - "description" - ], - "properties": { - "description": { - "description": "The message to display.", - "type": "string" - }, - "format": { - "type": "string", - "default": "text", - "description": "The format of the message, either a programming language, 'text' or 'html'." - } - } - } - } - } - } - ] - }, - "textConfigurationOptions": { - "oneOf": [ - { - "type": "object", - "description": "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", - "minProperties": 1, - "properties": { - "applyRounding": { - "description": "Apply rounding when comparing as float", - "type": "boolean" - }, - "caseInsensitive": { - "description": "Ignore case when comparing strings", - "type": "boolean" - }, - "ignoreWhitespace": { - "description": "Ignore trailing whitespace", - "type": "boolean" - }, - "normalizeTrailingNewlines": { - "description": "Normalize trailing newlines", - "type": "boolean" - }, - "roundTo": { - "description": "The number of decimals to round at, when applying the rounding on floats", - "type": "integer" - }, - "tryFloatingPoint": { - "description": "Try comparing text as floating point numbers", - "type": "boolean" }, "hideExpected": { - "description": "Hide the expected value in feedback (default: false), not recommended to use!", - "type": "boolean" + "anyOf": [ + { + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } + ] + }, + { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + ] } } }, @@ -3387,32 +5781,88 @@ "minProperties": 1, "properties": { "applyRounding": { - "description": "Apply rounding when comparing as float", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Apply rounding when comparing as float", + "type": "boolean" + } + ] }, "caseInsensitive": { - "description": "Ignore case when comparing strings", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Ignore case when comparing strings", + "type": "boolean" + } + ] }, "ignoreWhitespace": { - "description": "Ignore trailing whitespace", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Ignore trailing whitespace", + "type": "boolean" + } + ] }, "normalizeTrailingNewlines": { - "description": "Normalize trailing newlines", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Normalize trailing newlines", + "type": "boolean" + } + ] }, "roundTo": { - "description": "The number of decimals to round at, when applying the rounding on floats", - "type": "integer" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "The number of decimals to round at, when applying the rounding on floats", + "type": "integer" + } + ] }, "tryFloatingPoint": { - "description": "Try comparing text as floating point numbers", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Try comparing text as floating point numbers", + "type": "boolean" + } + ] }, "hideExpected": { - "description": "Hide the expected value in feedback (default: false), not recommended to use!", - "type": "boolean" + "anyOf": [ + { + "description": "The key of the parameter.", + "type": "string" + }, + { + "description": "Hide the expected value in feedback (default: false), not recommended to use!", + "type": "boolean" + } + ] } } } @@ -3505,41 +5955,102 @@ ] }, "textualType": { - "oneOf": [ + "anyOf": [ { - "description": "Simple textual value, converted to string.", - "type": [ - "string", - "number", - "integer", - "boolean" + "oneOf": [ + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "type": "object", + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter" + }, + "value": { + "description": "The key of the parameter.", + "type": "string" + } + } + } + } + } + } ] }, { - "type": "object", - "required": [ - "__tag__", - "value" - ], - "properties": { - "__tag__": { - "type": "string", - "description": "The tag used in the yaml", - "const": "!natural_language" + "oneOf": [ + { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] }, - "value": { + { "type": "object", - "additionalProperties": { - "description": "Simple textual value, converted to string.", - "type": [ - "string", - "number", - "integer", - "boolean" - ] + "required": [ + "__tag__", + "value" + ], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!natural_language" + }, + "value": { + "type": "object", + "additionalProperties": { + "description": "Simple textual value, converted to string.", + "type": [ + "string", + "number", + "integer", + "boolean" + ] + } + } } } - } + ] } ] }, diff --git a/tested/dsl/schema-strict.json b/tested/dsl/schema-strict.json index c486cf5e..7897cbf7 100644 --- a/tested/dsl/schema-strict.json +++ b/tested/dsl/schema-strict.json @@ -221,7 +221,7 @@ "type": "array", "minItems": 1, "items": { - "$ref": "#/definitions/script" + "$ref": "#/definitions/testcase" } }, "context": { @@ -339,75 +339,6 @@ "description": "Expected exit code for the run" } } - }, - "script": { - "type": "object", - "description": "An individual test (script) for a statement or expression", - "properties": { - "description": { - "$ref": "#/subDefinitions/message" - }, - "stdin": { - "description": "Stdin for this context", - "type": [ - "string", - "number", - "integer", - "boolean" - ] - }, - "arguments": { - "description": "Array of program call arguments", - "type": "array", - "items": { - "type": [ - "string", - "number", - "integer", - "boolean" - ] - } - }, - "statement": { - "description": "The statement to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "expression": { - "description": "The expression to evaluate.", - "$ref": "#/subDefinitions/expressionOrStatement" - }, - "exception": { - "description": "Expected exception message", - "$ref": "#/subDefinitions/exceptionChannel" - }, - "files": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - }, - "return": { - "description": "Expected return value", - "$ref": "#/subDefinitions/returnOutputChannel" - }, - "stderr": { - "description": "Expected output at stderr", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "stdout": { - "description": "Expected output at stdout", - "$ref": "#/subDefinitions/textOutputChannel" - }, - "file": { - "description": "Expected files generated by the submission.", - "$ref": "#/subDefinitions/fileOutputChannel" - }, - "exit_code": { - "type": "integer", - "description": "Expected exit code for the run" - } - } } }, "subDefinitions" : { diff --git a/tested/nat_translation.py b/tested/nat_translation.py index 65cfcfa5..3d365014 100644 --- a/tested/nat_translation.py +++ b/tested/nat_translation.py @@ -86,23 +86,71 @@ def construct_custom_tag_format(loader, tag_suffix, node): return {"__tag__": tag_suffix, "value": data} -def translate_yaml( - data: Any, translations: dict, language: str, env: Environment +def init_template( + template: dict, + translations: dict, + templates: dict, + parameters: dict, + language: str, + env: Environment, +) -> dict: + + # translate parameters before inserting them into the template + new_parameters = translate_yaml_and_initialize_templates( + parameters, translations, templates, parameters, language, env, True + ) + + # translate template and insert parameters + template = translate_yaml_and_initialize_templates( + template, translations, templates, new_parameters, language, env, True + ) + assert isinstance(template, dict) + return template + + +def translate_yaml_and_initialize_templates( + data: Any, + translations: dict, + templates: dict, + parameters: dict, + language: str, + env: Environment, + inside_templates: bool = False, ) -> Any: """ - This function will translate the multilingual object. + This function will translate the multilingual object and initialize the templates. :param data: The object to translate. :param translations: The merge of all found translations maps. + :param templates: The merge of all found templates. + :param parameters: The data passed to a template. This will only not be empty when a template is processed. :param language: The language to translate to. :param env: The Jinja-environment to use. + :param inside_templates: Indicator if a template is being processed. :return: The translated object. """ if isinstance(data, dict): - if "__tag__" in data and data["__tag__"] == "!natural_language": - value = data["value"] - assert language in value - return translate_yaml(value[language], translations, language, env) + if "__tag__" in data: + if data["__tag__"] == "!natural_language": + value = data["value"] + assert language in value + return translate_yaml_and_initialize_templates( + value[language], + translations, + templates, + parameters, + language, + env, + inside_templates, + ) + elif data["__tag__"] == "!parameter": + assert ( + data["value"] in parameters and inside_templates + ), "You are using the !parameter tag outside a template or the specified parameter does not exist." + return parameters[data["value"]] + + current_templates = data.pop("templates", {}) + templates = {**templates, **current_templates} current_translations = data.pop("translations", {}) for key, value in current_translations.items(): @@ -110,17 +158,108 @@ def translate_yaml( current_translations[key] = value[language] translations = {**translations, **current_translations} + if "template" in data or "parameters" in data: + assert ( + not parameters and not inside_templates + ), "A template was defined inside another template. This is not allowed!" + assert ( + "repeat" not in data + ), "A repeat was defined alongside a template. This is not allowed!" + if "template" in data: + name = data.pop("template") + assert name in templates + template = templates[name] + else: + raise ValueError("Found parameter without specifying template!") + template = init_template( + template, + translations, + templates, + data.pop("parameters", {}), + language, + env, + ) + + if data: + # Extra specifications in data will overwrite parts of the template. + data = translate_yaml_and_initialize_templates( + data, + translations, + templates, + parameters, + language, + env, + inside_templates, + ) + for key, value in data.items(): + template[key] = value + return template + + if "repeat" in data: + assert ( + not parameters and not inside_templates + ), "A repeat was defined inside another template. This is not allowed!" + repeat = data.pop("repeat") + assert "parameters" in repeat + parameters = repeat["parameters"] + assert isinstance(parameters, list) + + assert "template" in repeat + name = repeat.pop("template") + assert name in templates + template = templates[name] + + return [ + init_template( + template, + translations, + templates, + param_item, + language, + env, + ) + for param_item in parameters + ] + return { - key: translate_yaml(value, translations, language, env) + key: translate_yaml_and_initialize_templates( + value, + translations, + templates, + parameters, + language, + env, + inside_templates, + ) for key, value in data.items() } elif isinstance(data, list): - return [translate_yaml(item, translations, language, env) for item in data] + result = [] + for item in data: + has_repeat = isinstance(item, dict) and "repeat" in item + translated = translate_yaml_and_initialize_templates( + item, + translations, + templates, + parameters, + language, + env, + inside_templates, + ) + if has_repeat and isinstance(translated, list): + result.extend(translated) + else: + result.append(translated) + return result elif isinstance(data, str): + assert ( + len(set(translations.keys()).intersection(set(parameters.keys()))) == 0 + ), "Found a key in the translations map that is the same as inside a template. Please try to avoid this!" try: - return env.from_string(data).render(translations) + return env.from_string(data).render({**translations, **parameters}) except TemplateSyntaxError: return data + return data @@ -190,7 +329,9 @@ def apply_translations( validate_pre_dsl(parsed_yaml) enviroment = create_enviroment() - translated_data = translate_yaml(parsed_yaml, {}, language, enviroment) + translated_data = translate_yaml_and_initialize_templates( + parsed_yaml, {}, {}, {}, language, enviroment + ) missing_keys = TrackingUndefined.missing_keys messages = build_preprocessor_messages(missing_keys) diff --git a/tested/transform_json.py b/tested/transform_json.py index bab82272..a175e69a 100644 --- a/tested/transform_json.py +++ b/tested/transform_json.py @@ -1,10 +1,11 @@ +import copy import json import os import sys from pathlib import Path from typing import Any -SPECIAL_TYPES = ["expression", "programming_language", "oracle"] +SPECIAL_TYPES = ["expression", "programming_language", "oracle", "parameter"] def transform_non_strict(data: Any) -> Any: @@ -111,10 +112,16 @@ def flatten_one_of_stack(data: list) -> list: return new_one_stack -def make_tag_structure( - data: Any, data_with_inner_translations: Any = None, tag: str = "!natural_language" -) -> dict: +def make_parameter_type_structure(data: dict) -> dict: + return { + "anyOf": [ + {"type": "parameter", "description": "The key of the parameter."}, + data, + ] + } + +def make_tag_structure(data: dict, tag: str = "!natural_language") -> dict: base = { "type": "object", "required": ["__tag__", "value"], @@ -127,75 +134,75 @@ def make_tag_structure( }, } - if tag in ["!oracle", "!programming_language", "!expression"]: + if tag != "!natural_language": base["properties"]["value"] = data base["properties"]["value"]["type"] = "object" - if tag == "!expression": + if tag in ["!expression", "!parameter"]: base["properties"]["value"]["type"] = "string" - elif tag == "!natural_language": + else: base["properties"]["value"] = { "type": "object", "additionalProperties": data, } - if data_with_inner_translations is not None: - return {"oneOf": [data_with_inner_translations, base]} return base def make_translations_map() -> dict: - return {"type": "object", "description": "Define translations in the global scope."} + return {"type": "object", "description": "Define translations."} + +def make_templates_map() -> dict: + return { + "type": "object", + "description": "Define templates.", + "additionalProperties": { + "$ref": "#/definitions/testcase_without_templates", + }, + } -def transform_json_for_preprocessor_validation(data: Any, in_sub_def: bool) -> Any: + +def transform_json_for_preprocessor_validation(data: Any) -> Any: """ - This function is responsible for transforming the JSON-schema such that translations are supported. + This function will start with the result of add_parameter_type. + It is responsible for transforming the JSON-schema such that translations and templates are supported. It also uses a special structure for tags in the YAML that is used in the preprocessing step. :param data: The data to transform. - :param in_sub_def: Indicates if the sub-definition are being processed. :return: The transformed data. """ if isinstance(data, dict): - # Standard creation of the special tag structure for translations. + # Add the translations and templates maps + targets = ["_rootObject", "tab", "unit", "context", "case"] + for target in targets: + if target in data and "properties" in data[target]: + data[target]["properties"]["translations"] = make_translations_map() + data[target]["properties"]["templates"] = make_templates_map() + new_data = { - key: transform_json_for_preprocessor_validation( - value, in_sub_def or key == "subDefinitions" + key: ( + transform_json_for_preprocessor_validation(value) + if key != "translations" and key != "templates" + else value ) for key, value in data.items() } - if ( - "type" in data - and (data["type"] != "object" or in_sub_def) - and data["type"] not in ["boolean", "integer"] - ): + + # Standard creation of the special tag structure for translations and templates. + if "type" in data and data["type"] not in ["boolean", "integer", "number"]: + # Adding support for the other tags. if data["type"] in SPECIAL_TYPES: tag = data.pop("type") new_data.pop("type") # translations applied to inner part - tag_data_with_inner_translations = make_tag_structure( - new_data, - tag=f"!{tag}", - ) + new_data = make_tag_structure(new_data, tag=f"!{tag}") # translations not applied to inner part - tag_data = make_tag_structure( - data, - tag=f"!{tag}", - ) - return make_tag_structure( - data=tag_data, - data_with_inner_translations=tag_data_with_inner_translations, - ) - - return make_tag_structure(data, new_data) - data = new_data + data = make_tag_structure(data, tag=f"!{tag}") - # Add the translations maps - targets = ["_rootObject", "tab", "unit", "context", "case"] - for target in targets: - if target in data and "properties" in data[target]: - data[target]["properties"]["translations"] = make_translations_map() + result = make_tag_structure(data) + return {"oneOf": [new_data, result]} + data = new_data # Flatten the oneOf structures if "oneOf" in data: @@ -207,6 +214,21 @@ def transform_json_for_preprocessor_validation(data: Any, in_sub_def: bool) -> A data["yamlValueOrPythonExpression"] = { "oneOf": [ {"$ref": "#/subDefinitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter", + }, + "value": { + "type": "string", + "description": "The key of the parameter.", + }, + }, + }, { "type": "object", "required": ["__tag__", "value"], @@ -239,14 +261,95 @@ def transform_json_for_preprocessor_validation(data: Any, in_sub_def: bool) -> A "$ref" in data and data["$ref"] == "#/subDefinitions/yamlValueOrPythonExpression" ): - data = make_tag_structure(data, data) + data = { + "oneOf": [ + {"$ref": "#/subDefinitions/yamlValueOrPythonExpression"}, + make_tag_structure(data), + ] + } return data if isinstance(data, list): - return [ - transform_json_for_preprocessor_validation(value, in_sub_def) - for value in data - ] + return [transform_json_for_preprocessor_validation(value) for value in data] + return data + + +def add_templates(data: dict) -> dict: + """ + This function will start by a new definition for a single testcase and add templates to the existing definition. + The new definition for a testcase will be used to specify a testcase inside a template. + + :param data: The data to transform. + :return: The transformed data. + """ + assert "definitions" in data and isinstance(data["definitions"], dict) + # Addition for adding templates to the json-schema + testcase = data["definitions"].get("testcase", {}) + if testcase: + assert "properties" in testcase + data["definitions"]["testcase_without_templates"] = copy.deepcopy(testcase) + + testcase["properties"]["template"] = { + "type": "string", + "description": "Name of the template to insert.", + } + testcase["properties"]["parameters"] = { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + } + testcase["properties"]["repeat"] = { + "type": "object", + "description": "A loop to generate test cases with the given parameters and template.", + "required": ["template", "parameters"], + "properties": { + "template": { + "type": "string", + "description": "Name of the template to insert.", + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + }, + }, + } + + data["definitions"]["testcase"] = testcase + return data + + +def add_parameter_type(data: Any) -> Any: + """ + This function will start with the result of add_templates and add the potential to use the type + parameter in specific places. + + :param data: The data to transform. + :return: The transformed data. + """ + if isinstance(data, dict): + if "type" in data: + type_value = data["type"] + if ( + isinstance(type_value, list) + and any(t in ["boolean", "integer", "number"] for t in type_value) + ) or type_value in ["boolean", "integer", "number"]: + # Adding support for "!parameter" tag. + return make_parameter_type_structure(data) + + if "arguments" in data: + data["arguments"] = make_parameter_type_structure(data["arguments"]) + return {k: add_parameter_type(v) for k, v in data.items()} + + if isinstance(data, list): + return [add_parameter_type(value) for value in data] return data @@ -272,7 +375,11 @@ def transform_json(json_file: Path, multilingual: bool, ide: bool): result = transform_non_strict(json_stream) file_name = "schema.json" else: - result = transform_json_for_preprocessor_validation(json_stream, False) + stream_with_templates = add_templates(json_stream) + stream_with_templates_and_param = add_parameter_type(stream_with_templates) + result = transform_json_for_preprocessor_validation( + stream_with_templates_and_param + ) if ide: result = transform_ide(result) file_name = "multilingual-schema.json" diff --git a/tests/test_json_schema_transformer.py b/tests/test_json_schema_transformer.py index 3811f167..244fb014 100644 --- a/tests/test_json_schema_transformer.py +++ b/tests/test_json_schema_transformer.py @@ -4,6 +4,8 @@ import tested from tested.transform_json import ( + add_parameter_type, + add_templates, transform_ide, transform_json, transform_json_for_preprocessor_validation, @@ -173,14 +175,16 @@ def test_transform_executed_correct(mocker: MockerFixture): mocker.mock_open(read_data=content).return_value for content in [ """ -{ - "files": { - "description": "A list of files used in the test suite.", - "type": "array", - "items": { - "$ref": "#/subDefinitions/file" - } - } +{ + "definitions": { + "files": { + "description": "A list of files used in the test suite.", + "type": "array", + "items": { + "$ref": "#/subDefinitions/file" + } + } + } }""" ] ] @@ -192,7 +196,7 @@ def test_transform_executed_correct(mocker: MockerFixture): transform_json(Path("schema.json"), True, False) - assert s.call_count == 6 + assert s.call_count == 7 # Check if the file was opened for writing mock_opener.assert_any_call( @@ -232,7 +236,7 @@ def test_nat_lang_json_schema_structure(): ] } - result = transform_json_for_preprocessor_validation(json_schema, False) + result = transform_json_for_preprocessor_validation(json_schema) assert result == json_schema_expected @@ -340,7 +344,7 @@ def test_prog_lang_json_schema_structure(): } } - result = transform_json_for_preprocessor_validation(json_schema, True) + result = transform_json_for_preprocessor_validation(json_schema) assert result == json_schema_expected @@ -440,7 +444,7 @@ def test_json_schema_oracle(): ] } - result = transform_json_for_preprocessor_validation(json_schema, True) + result = transform_json_for_preprocessor_validation(json_schema) assert result == json_schema_expected @@ -463,6 +467,21 @@ def test_json_schema_expression(): "yamlValueOrPythonExpression": { "oneOf": [ {"$ref": "#/subDefinitions/yamlValue"}, + { + "type": "object", + "required": ["__tag__", "value"], + "properties": { + "__tag__": { + "type": "string", + "description": "The tag used in the yaml", + "const": "!parameter", + }, + "value": { + "type": "string", + "description": "The key of the parameter.", + }, + }, + }, { "type": "object", "required": ["__tag__", "value"], @@ -483,7 +502,125 @@ def test_json_schema_expression(): } } - result = transform_json_for_preprocessor_validation(json_schema, True) + result = transform_json_for_preprocessor_validation(json_schema) + assert result == json_schema_expected + + +def test_add_templates(): + json_schema = { + "definitions": { + "testcase": { + "properties": { + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement", + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement", + }, + } + } + } + } + + json_schema_expected = { + "definitions": { + "testcase": { + "properties": { + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement", + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement", + }, + "template": { + "type": "string", + "description": "Name of the template to insert.", + }, + "parameters": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + "repeat": { + "type": "object", + "description": "A certain loop that will generate test cases with the given parameters and template.", + "required": ["template", "parameters"], + "properties": { + "template": { + "type": "string", + "description": "Name of the template to insert.", + }, + "parameters": { + "type": "array", + "items": { + "type": "object", + "description": "The parameters that are to be inserted into the template.", + "additionalProperties": { + "$ref": "#/subDefinitions/yamlValueOrPythonExpression" + }, + }, + }, + }, + }, + } + }, + "testcase_without_templates": { + "properties": { + "statement": { + "description": "The statement to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement", + }, + "expression": { + "description": "The expression to evaluate.", + "$ref": "#/subDefinitions/expressionOrStatement", + }, + } + }, + } + } + + result = add_templates(json_schema) + assert result == json_schema_expected + + +def test_add_parameters_type(): + json_schema = { + "stdin": { + "description": "Stdin for this context", + "type": ["string", "number", "integer", "boolean"], + }, + "exit_code": { + "type": "integer", + "description": "Expected exit code for the run", + }, + } + + json_schema_expected = { + "stdin": { + "anyOf": [ + {"type": "parameter", "description": "The key of the parameter."}, + { + "description": "Stdin for this context", + "type": ["string", "number", "integer", "boolean"], + }, + ] + }, + "exit_code": { + "anyOf": [ + {"type": "parameter", "description": "The key of the parameter."}, + {"type": "integer", "description": "Expected exit code for the run"}, + ] + }, + } + + result = add_parameter_type(json_schema) + print(result) assert result == json_schema_expected @@ -501,7 +638,7 @@ def test_json_schema_yaml_value(): }, } - result = transform_json_for_preprocessor_validation(json_schema, False) + result = transform_json_for_preprocessor_validation(json_schema) assert result == json_schema_expected diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py index 2357ca2a..71305c54 100644 --- a/tests/test_preprocess_dsl.py +++ b/tests/test_preprocess_dsl.py @@ -16,7 +16,7 @@ create_enviroment, parse_yaml, translate_file, - translate_yaml, + translate_yaml_and_initialize_templates, validate_pre_dsl, ) @@ -25,7 +25,9 @@ def validate_natural_translate(yaml_str: str, translated_yaml_str: str): enviroment = create_enviroment() yaml_object = parse_yaml(yaml_str) validate_pre_dsl(yaml_object) - translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment) + translated_dsl = translate_yaml_and_initialize_templates( + yaml_object, {}, {}, {}, "en", enviroment + ) translated_yaml = convert_to_yaml(translated_dsl) assert translated_yaml.strip() == translated_yaml_str @@ -353,6 +355,206 @@ def test_natural_translate_io_test(): validate_natural_translate(yaml_str, translated_yaml_str) +def test_normal_template(): + yaml_str = """ +tabs: + - tab: 'animals' + templates: + animal_IO: + expression: + java: "Submission.animals({{ input }})" + python: "animals({{ input }})" + return: "{{ output }}" + testcases: + - template: "animal_IO" + parameters: + input: "Sardines" + output: "fish" + - template: "animal_IO" + parameters: + input: "Whale" + output: "mammal" + return: "mammal (not a fish)" + """.strip() + translated_yaml_str = """ +tabs: +- tab: animals + testcases: + - expression: + java: Submission.animals(Sardines) + python: animals(Sardines) + return: fish + - expression: + java: Submission.animals(Whale) + python: animals(Whale) + return: mammal (not a fish) + """.strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_normal_repeat(): + yaml_str = """ +templates: + one_name: + stdin: |- + 1 + {{ name }} + stdout: "{{ username }}" +units: +- unit: IO + scripts: + - template: one_name + parameters: + name: Graham Chapman + username: gchap + - repeat: + template: one_name + parameters: + - name: John Cleese + username: jclee + - name: Terry Gilliam + username: tgill + - name: Eric Idle + username: eidle + - name: Terry Jones + username: tjone + - name: Michael Palin + username: mpali + """.strip() + translated_yaml_str = """ +units: +- unit: IO + scripts: + - stdin: '1 + + Graham Chapman' + stdout: gchap + - stdin: '1 + + John Cleese' + stdout: jclee + - stdin: '1 + + Terry Gilliam' + stdout: tgill + - stdin: '1 + + Eric Idle' + stdout: eidle + - stdin: '1 + + Terry Jones' + stdout: tjone + - stdin: '1 + + Michael Palin' + stdout: mpali + """.strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_templates_with_translations_wrong(): + yaml_str = """ +translations: + animals: + en: "animals" + nl: "dieren" +tabs: + - tab: '{{ animals }}' + templates: + animal_IO: + expression: + java: "Submission.{{ animals }}({{ animals }})" + python: "{{animals}}({{ animals }})" + return: "{{ output }}" + testcases: + - template: "animal_IO" + parameters: + animals: "Sardines" + output: "fish" + """.strip() + try: + validate_natural_translate(yaml_str, "") + except AssertionError as e: + assert ( + repr(e) + == "AssertionError('Found a key in the translations map that is the same as inside a template. Please try to avoid this!')" + ) + else: + assert False + + +def test_templates_with_translations_correct(): + yaml_str = """ +translations: + animals: + en: "animals" + nl: "dieren" +tabs: + - tab: '{{ animals }}' + templates: + animal_IO: + expression: + java: "Submission.{{ animals }}({{ input }})" + python: "{{animals}}({{ input }})" + return: "{{ output }}" + testcases: + - template: "animal_IO" + parameters: + input: "Sardines" + output: "fish" + """.strip() + translated_yaml_str = """ +tabs: +- tab: animals + testcases: + - expression: + java: Submission.animals(Sardines) + python: animals(Sardines) + return: fish + """.strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + +def test_non_string_parameter(): + yaml_str = """ +templates: + calculator_IO: + expression: "calculator('{{ expr }}')" + return: !parameter "result" + calculator_IO_non_int: + expression: "calculator('{{ expr }}', string)" + return: "{{ result }}" +tabs: + - tab: 'calculator' + testcases: + - template: "calculator_IO" + parameters: + expr: "2 + 10" + result: 12 + - template: "calculator_IO" + parameters: + expr: "4 * 13" + result: 52 + - template: "calculator_IO_non_int" + parameters: + expr: "4 * 13" + result: 52 + """.strip() + translated_yaml_str = """ +tabs: +- tab: calculator + testcases: + - expression: calculator('2 + 10') + return: 12 + - expression: calculator('4 * 13') + return: 52 + - expression: calculator('4 * 13', string) + return: '52' + """.strip() + validate_natural_translate(yaml_str, translated_yaml_str) + + def test_validation(): yaml_str = """ translations: @@ -394,7 +596,9 @@ def test_validation(): validate_pre_dsl(yaml_object) enviroment = create_enviroment() - translated_data = translate_yaml(yaml_object, {}, "en", enviroment) + translated_data = translate_yaml_and_initialize_templates( + yaml_object, {}, {}, {}, "en", enviroment + ) translated_yaml_string = convert_to_yaml(translated_data) _validate_dsl(_parse_yaml(translated_yaml_string)) @@ -569,7 +773,9 @@ def test_to_yaml_object(): environment = create_enviroment() parsed_yaml = parse_yaml(yaml_str) - translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment) + translated_dsl = translate_yaml_and_initialize_templates( + parsed_yaml, {}, {}, {}, "en", environment + ) translated_yaml_string = convert_to_yaml(translated_dsl) yaml_object = _parse_yaml(translated_yaml_string) assert isinstance(yaml_object, dict)