From d1a544baf583bcb1db0620ad9135d0a924ed3426 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 19:35:13 +0100
Subject: [PATCH 001/111] Made first version for translation using
 !natural_language

---
 tested/dsl/translate_parser.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 0aa48574..ad8013af 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -83,14 +83,16 @@ class TestedType:
 class ExpressionString(str):
     pass
 
-
 class ReturnOracle(dict):
     pass
 
+class NaturalLanguageMap(dict):
+    pass
+
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
-    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle
+    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | NaturalLanguageMap
 )
 
 
@@ -137,6 +139,12 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     ), f"A custom oracle must be an object, got {result} which is a {type(result)}."
     return ReturnOracle(result)
 
+def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
+    return NaturalLanguageMap(result)
 
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
@@ -148,6 +156,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
             yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
     yaml.add_constructor("!expression", _expression_string, loader)
     yaml.add_constructor("!oracle", _return_oracle, loader)
+    yaml.add_constructor("!natural_language", _natural_language_map, loader)
 
     try:
         return yaml.load(yaml_stream, loader)

From d0cf7de8e2b16c12ca005588e5a6aa9465ec8e39 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 19:43:36 +0100
Subject: [PATCH 002/111] forgot to push actual file

---
 tested/nat_translation.py | 182 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 182 insertions(+)
 create mode 100644 tested/nat_translation.py

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
new file mode 100644
index 00000000..5e4f7ff5
--- /dev/null
+++ b/tested/nat_translation.py
@@ -0,0 +1,182 @@
+import sys
+from typing import cast
+
+import yaml
+from tested.dsl.translate_parser import _parse_yaml, YamlObject, YamlDict, ReturnOracle, \
+    ExpressionString, _validate_testcase_combinations, NaturalLanguageMap
+
+
+def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
+    _validate_testcase_combinations(testcase)
+
+    key_to_set = "statement" if "statement" in testcase else "expression"
+    if (expr_stmt := testcase.get(key_to_set)) is not None:
+        # Must use !natural_language
+        if isinstance(expr_stmt, NaturalLanguageMap):
+            assert language in expr_stmt
+            testcase[key_to_set] = expr_stmt[language]
+    else:
+        if (stdin_stmt := testcase.get("stdin")) is not None:
+            if isinstance(stdin_stmt, dict):
+                assert language in stdin_stmt
+                testcase["stdin"] = stdin_stmt[language]
+
+        arguments = testcase.get("arguments", [])
+        if isinstance(arguments, dict):
+            assert language in arguments
+            testcase["arguments"] = arguments[language]
+
+    if (stdout := testcase.get("stdout")) is not None:
+        # Must use !natural_language
+        if isinstance(stdout, NaturalLanguageMap):
+            assert language in stdout
+            testcase["stdout"] = stdout[language]
+        elif isinstance(stdout["data"], dict):
+            assert language in stdout["data"]
+            testcase["stdout"]["data"] = stdout["data"][language]
+    if (file := testcase.get("file")) is not None:
+        # Must use !natural_language
+        if isinstance(file, NaturalLanguageMap):
+            assert language in file
+            testcase["file"] = file[language]
+    if (stderr := testcase.get("stderr")) is not None:
+        # Must use !natural_language
+        if isinstance(stderr, NaturalLanguageMap):
+            assert language in stderr
+            testcase["stderr"] = stderr[language]
+        elif isinstance(stderr["data"], dict):
+            assert language in stderr["data"]
+            testcase["stderr"]["data"] = stderr["data"][language]
+    if (exception := testcase.get("exception")) is not None:
+        if isinstance(exception, NaturalLanguageMap):
+            assert language in exception
+            testcase["exception"] = exception[language]
+        elif isinstance(exception["message"], dict):
+            assert language in exception["message"]
+            testcase["exception"]["message"] = exception["message"][language]
+
+    if (result := testcase.get("return")) is not None:
+        if isinstance(result, ReturnOracle):
+            arguments = result.get("arguments", [])
+            if isinstance(arguments, dict):
+                assert language in arguments
+                result["arguments"] = arguments[language]
+
+            value = result.get("value")
+            # Must use !natural_language
+            if isinstance(value, NaturalLanguageMap):
+                assert language in value
+                result["value"] = value[language]
+
+            testcase["return"] = result
+        elif isinstance(result, NaturalLanguageMap):
+            # Must use !natural_language
+            assert language in result
+            testcase["return"] = result[language]
+
+    if (description := testcase.get("description")) is not None:
+        # Must use !natural_language
+        if isinstance(description, NaturalLanguageMap):
+            assert language in description
+            testcase["description"] = description[language]
+        elif isinstance(description, dict) and isinstance(description["description"], dict):
+            assert language in description["description"]
+            description["description"] = description["description"][language]
+            testcase["description"] = description
+
+    return testcase
+
+def translate_testcases(testcases: list, language: str) -> list:
+    result = []
+    for testcase in testcases:
+        assert isinstance(testcase, dict)
+        result.append(translate_testcase(testcase, language))
+
+    return result
+
+def translate_contexts(contexts: list, language: str) -> list:
+    result = []
+    for context in contexts:
+        assert isinstance(context, dict)
+        raw_testcases = context.get("script", context.get("testcases"))
+        assert isinstance(raw_testcases, list)
+        result.append(translate_testcases(raw_testcases, language))
+
+    return result
+
+def translate_tab(tab: YamlDict, language: str) -> YamlDict:
+    key_to_set = "units" if "units" in tab else "tab"
+    name = tab.get(key_to_set)
+
+    if isinstance(name, dict):
+        assert language in name
+        tab[key_to_set] = name[language]
+
+    # The tab can have testcases or contexts.
+    if "contexts" in tab:
+        assert isinstance(tab["contexts"], list)
+        tab["contexts"] = translate_contexts(tab["contexts"], language)
+    elif "cases" in tab:
+        assert "unit" in tab
+        # We have testcases N.S. / contexts O.S.
+        assert isinstance(tab["cases"], list)
+        tab["cases"] = translate_contexts(tab["cases"], language)
+    elif "testcases" in tab:
+        # We have scripts N.S. / testcases O.S.
+        assert "tab" in tab
+        assert isinstance(tab["testcases"], list)
+        tab["testcases"] = translate_testcases(tab["testcases"], language)
+    else:
+        print(tab)
+        assert "scripts" in tab
+        assert isinstance(tab["scripts"], list)
+        tab["scripts"] = translate_testcases(tab["scripts"], language)
+    return tab
+
+def translate_tabs(dsl_list: list, language: str) -> list:
+    result = []
+    for tab in dsl_list:
+        assert isinstance(tab, dict)
+        result.append(translate_tab(tab, language))
+
+    return result
+
+def translate_dsl(dsl_object: YamlObject, language:str) -> YamlObject:
+    if isinstance(dsl_object, list):
+        return translate_tabs(dsl_object, language)
+    else:
+        assert isinstance(dsl_object, dict)
+        key_to_set = "units" if "units" in dsl_object else "tabs"
+        tab_list = dsl_object.get(key_to_set)
+        assert isinstance(tab_list, list)
+        dsl_object[key_to_set] = translate_tabs(tab_list, language)
+        return dsl_object
+
+def parse_yaml(yaml_path:str) -> YamlObject:
+    with open(yaml_path, 'r') as stream:
+        result = _parse_yaml(stream.read())
+
+    return result
+
+def convert_to_yaml(yaml_object: YamlObject) -> str:
+    def oracle_representer(dumper, data):
+        return dumper.represent_mapping('!oracle', data)
+
+    def expression_representer(dumper, data):
+        return dumper.represent_scalar('!expression', data)
+
+    # Register the representer for the ReturnOracle object
+    yaml.add_representer(ReturnOracle, oracle_representer)
+    yaml.add_representer(ExpressionString, expression_representer)
+    return yaml.dump(yaml_object, sort_keys=False)
+
+if __name__ == '__main__':
+    n = len(sys.argv)
+    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+
+    path = sys.argv[1]
+    lang = sys.argv[2]
+    new_yaml = parse_yaml(path)
+    translated_dsl = translate_dsl(new_yaml, lang)
+    print(convert_to_yaml(translated_dsl))
+

From 0334e1ff32eeb7f78f2639020041eac1ac9fa1b6 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 19:44:04 +0100
Subject: [PATCH 003/111] fixed linting

---
 tested/dsl/translate_parser.py | 15 +++++++++++++-
 tested/nat_translation.py      | 36 ++++++++++++++++++++++++----------
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index ad8013af..9701a227 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -83,16 +83,27 @@ class TestedType:
 class ExpressionString(str):
     pass
 
+
 class ReturnOracle(dict):
     pass
 
+
 class NaturalLanguageMap(dict):
     pass
 
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
-    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | NaturalLanguageMap
+    YamlDict
+    | list
+    | bool
+    | float
+    | int
+    | str
+    | None
+    | ExpressionString
+    | ReturnOracle
+    | NaturalLanguageMap
 )
 
 
@@ -139,6 +150,7 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     ), f"A custom oracle must be an object, got {result} which is a {type(result)}."
     return ReturnOracle(result)
 
+
 def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
     result = _parse_yaml_value(loader, node)
     assert isinstance(
@@ -146,6 +158,7 @@ def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLangua
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
+
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
     Parse a string or stream to YAML.
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 5e4f7ff5..76ba5388 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,8 +2,15 @@
 from typing import cast
 
 import yaml
-from tested.dsl.translate_parser import _parse_yaml, YamlObject, YamlDict, ReturnOracle, \
-    ExpressionString, _validate_testcase_combinations, NaturalLanguageMap
+from tested.dsl.translate_parser import (
+    _parse_yaml,
+    YamlObject,
+    YamlDict,
+    ReturnOracle,
+    ExpressionString,
+    _validate_testcase_combinations,
+    NaturalLanguageMap,
+)
 
 
 def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
@@ -79,13 +86,16 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(description, NaturalLanguageMap):
             assert language in description
             testcase["description"] = description[language]
-        elif isinstance(description, dict) and isinstance(description["description"], dict):
+        elif isinstance(description, dict) and isinstance(
+            description["description"], dict
+        ):
             assert language in description["description"]
             description["description"] = description["description"][language]
             testcase["description"] = description
 
     return testcase
 
+
 def translate_testcases(testcases: list, language: str) -> list:
     result = []
     for testcase in testcases:
@@ -94,6 +104,7 @@ def translate_testcases(testcases: list, language: str) -> list:
 
     return result
 
+
 def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
@@ -104,6 +115,7 @@ def translate_contexts(contexts: list, language: str) -> list:
 
     return result
 
+
 def translate_tab(tab: YamlDict, language: str) -> YamlDict:
     key_to_set = "units" if "units" in tab else "tab"
     name = tab.get(key_to_set)
@@ -133,6 +145,7 @@ def translate_tab(tab: YamlDict, language: str) -> YamlDict:
         tab["scripts"] = translate_testcases(tab["scripts"], language)
     return tab
 
+
 def translate_tabs(dsl_list: list, language: str) -> list:
     result = []
     for tab in dsl_list:
@@ -141,7 +154,8 @@ def translate_tabs(dsl_list: list, language: str) -> list:
 
     return result
 
-def translate_dsl(dsl_object: YamlObject, language:str) -> YamlObject:
+
+def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
     if isinstance(dsl_object, list):
         return translate_tabs(dsl_object, language)
     else:
@@ -152,25 +166,28 @@ def translate_dsl(dsl_object: YamlObject, language:str) -> YamlObject:
         dsl_object[key_to_set] = translate_tabs(tab_list, language)
         return dsl_object
 
-def parse_yaml(yaml_path:str) -> YamlObject:
-    with open(yaml_path, 'r') as stream:
+
+def parse_yaml(yaml_path: str) -> YamlObject:
+    with open(yaml_path, "r") as stream:
         result = _parse_yaml(stream.read())
 
     return result
 
+
 def convert_to_yaml(yaml_object: YamlObject) -> str:
     def oracle_representer(dumper, data):
-        return dumper.represent_mapping('!oracle', data)
+        return dumper.represent_mapping("!oracle", data)
 
     def expression_representer(dumper, data):
-        return dumper.represent_scalar('!expression', data)
+        return dumper.represent_scalar("!expression", data)
 
     # Register the representer for the ReturnOracle object
     yaml.add_representer(ReturnOracle, oracle_representer)
     yaml.add_representer(ExpressionString, expression_representer)
     return yaml.dump(yaml_object, sort_keys=False)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     n = len(sys.argv)
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
 
@@ -179,4 +196,3 @@ def expression_representer(dumper, data):
     new_yaml = parse_yaml(path)
     translated_dsl = translate_dsl(new_yaml, lang)
     print(convert_to_yaml(translated_dsl))
-

From c1114bcae5d7934189f9dcdf3294986acc881465 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 20:16:03 +0100
Subject: [PATCH 004/111] fixed pyright issue

---
 tested/nat_translation.py | 41 ++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 76ba5388..4a4a024b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,5 +1,4 @@
 import sys
-from typing import cast
 
 import yaml
 from tested.dsl.translate_parser import (
@@ -38,9 +37,12 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(stdout, NaturalLanguageMap):
             assert language in stdout
             testcase["stdout"] = stdout[language]
-        elif isinstance(stdout["data"], dict):
-            assert language in stdout["data"]
-            testcase["stdout"]["data"] = stdout["data"][language]
+        elif isinstance(stdout, dict):
+            data = stdout["data"]
+            if isinstance(data, dict):
+                assert language in data
+                stdout["data"] = data[language]
+                testcase["stdout"] = stdout
     if (file := testcase.get("file")) is not None:
         # Must use !natural_language
         if isinstance(file, NaturalLanguageMap):
@@ -51,16 +53,23 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(stderr, NaturalLanguageMap):
             assert language in stderr
             testcase["stderr"] = stderr[language]
-        elif isinstance(stderr["data"], dict):
-            assert language in stderr["data"]
-            testcase["stderr"]["data"] = stderr["data"][language]
+        elif isinstance(stderr, dict):
+            data = stderr["data"]
+            if isinstance(data, dict):
+                assert language in data
+                stderr["data"] = data[language]
+                testcase["stderr"] = stderr
+
     if (exception := testcase.get("exception")) is not None:
         if isinstance(exception, NaturalLanguageMap):
             assert language in exception
             testcase["exception"] = exception[language]
-        elif isinstance(exception["message"], dict):
-            assert language in exception["message"]
-            testcase["exception"]["message"] = exception["message"][language]
+        elif isinstance(exception, dict):
+            message = exception["message"]
+            if isinstance(message, dict):
+                assert language in message
+                exception["message"] = message[language]
+                testcase["exception"] = exception
 
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
@@ -86,12 +95,12 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(description, NaturalLanguageMap):
             assert language in description
             testcase["description"] = description[language]
-        elif isinstance(description, dict) and isinstance(
-            description["description"], dict
-        ):
-            assert language in description["description"]
-            description["description"] = description["description"][language]
-            testcase["description"] = description
+        elif isinstance(description, dict):
+            dd = description["description"]
+            if isinstance(dd, dict):
+                assert language in dd
+                description["description"] = dd[language]
+                testcase["description"] = description
 
     return testcase
 

From c61b5634fdd946fab4584d99132c5b60ec996833 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 21:21:53 +0100
Subject: [PATCH 005/111] add test for unit-test

---
 tested/nat_translation.py | 36 ++++++++++++----------
 tests/test_dsl_yaml.py    | 63 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 82 insertions(+), 17 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 4a4a024b..9f798c80 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,14 +1,15 @@
 import sys
 
 import yaml
+
 from tested.dsl.translate_parser import (
-    _parse_yaml,
-    YamlObject,
-    YamlDict,
-    ReturnOracle,
     ExpressionString,
-    _validate_testcase_combinations,
     NaturalLanguageMap,
+    ReturnOracle,
+    YamlDict,
+    YamlObject,
+    _parse_yaml,
+    _validate_testcase_combinations,
 )
 
 
@@ -118,9 +119,11 @@ def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
-        raw_testcases = context.get("script", context.get("testcases"))
+        key_to_set = "script" if "script" in context else "testcases"
+        raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
-        result.append(translate_testcases(raw_testcases, language))
+        context[key_to_set] = translate_testcases(raw_testcases, language)
+        result.append(context)
 
     return result
 
@@ -196,12 +199,13 @@ def expression_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-if __name__ == "__main__":
-    n = len(sys.argv)
-    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
-
-    path = sys.argv[1]
-    lang = sys.argv[2]
-    new_yaml = parse_yaml(path)
-    translated_dsl = translate_dsl(new_yaml, lang)
-    print(convert_to_yaml(translated_dsl))
+# if __name__ == "__main__":
+#     n = len(sys.argv)
+#     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+#
+#     path = sys.argv[1]
+#     lang = sys.argv[2]
+#     new_yaml = parse_yaml(path)
+#     print(new_yaml)
+#     translated_dsl = translate_dsl(new_yaml, lang)
+#     print(convert_to_yaml(translated_dsl))
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 3caef72d..584320ef 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -20,7 +20,8 @@
     StringTypes,
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
-from tested.dsl.translate_parser import load_schema_validator
+from tested.dsl.translate_parser import load_schema_validator, _parse_yaml
+from tested.nat_translation import translate_dsl, convert_to_yaml
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1319,3 +1320,63 @@ def test_editor_json_schema_is_valid():
     validator = load_schema_validator("schema.json")
     assert isinstance(validator.schema, dict)
     validator.check_schema(validator.schema)
+
+def test_natural_translate_unit_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
+    yaml_str = """- tab:
+    en: "counting"
+    nl: "tellen"
+  contexts:
+    - testcases:
+      - statement: !natural_language
+          en: 'result = trying(10)'
+          nl: 'resultaat = proberen(10)'
+      - expression: !natural_language
+          en: 'count_words(result)'
+          nl: 'tel_woorden(resultaat)'
+        return: !natural_language
+          en: 'The result is 10'
+          nl: 'Het resultaat is 10'
+      - expression: !natural_language
+          en: !expression "count"
+          nl: !expression "tellen"
+        return: !natural_language
+          en: 'count'
+          nl: 'tellen'
+      - expression: 'ok(10)'
+        return: !oracle
+          value: !natural_language
+            en: "The value 10 is OK!"
+            nl: "De waarde 10 is OK!"
+          oracle: "custom_check"
+          file: "test.py"
+          name: "evaluate_test"
+          arguments:
+            en: ["The value", "is OK!", "is not OK!"]
+            nl: ["De waarde", "is OK!", "is niet OK!"]    
+    """
+    translated_yaml_str = """- tab: counting
+  contexts:
+  - testcases:
+    - statement: result = trying(10)
+    - expression: count_words(result)
+      return: The result is 10
+    - expression: !expression 'count'
+      return: count
+    - expression: ok(10)
+      return: !oracle
+        value: The value 10 is OK!
+        oracle: custom_check
+        file: test.py
+        name: evaluate_test
+        arguments:
+        - The value
+        - is OK!
+        - is not OK!
+"""
+    parsed_yaml = _parse_yaml(yaml_str)
+    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml == translated_yaml_str

From 145deaec59393e5b135efad4073489f6a293b530 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 12 Dec 2024 18:01:16 +0100
Subject: [PATCH 006/111] Fixed some bugs and wrote another test for io

---
 tested/nat_translation.py |  17 +++--
 tests/test_dsl_yaml.py    | 135 +++++++++++++++++++++++++++++++++++---
 2 files changed, 137 insertions(+), 15 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 9f798c80..b011496a 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -9,6 +9,7 @@
     YamlDict,
     YamlObject,
     _parse_yaml,
+    _validate_dsl,
     _validate_testcase_combinations,
 )
 
@@ -119,17 +120,19 @@ def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
-        key_to_set = "script" if "script" in context else "testcases"
-        raw_testcases = context.get(key_to_set)
-        assert isinstance(raw_testcases, list)
-        context[key_to_set] = translate_testcases(raw_testcases, language)
+        print(f"context: {context}")
+        if "script" in context or "testcases" in context:
+            key_to_set = "script" if "script" in context else "testcases"
+            raw_testcases = context.get(key_to_set)
+            assert isinstance(raw_testcases, list)
+            context[key_to_set] = translate_testcases(raw_testcases, language)
         result.append(context)
 
     return result
 
 
 def translate_tab(tab: YamlDict, language: str) -> YamlDict:
-    key_to_set = "units" if "units" in tab else "tab"
+    key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
 
     if isinstance(name, dict):
@@ -208,4 +211,6 @@ def expression_representer(dumper, data):
 #     new_yaml = parse_yaml(path)
 #     print(new_yaml)
 #     translated_dsl = translate_dsl(new_yaml, lang)
-#     print(convert_to_yaml(translated_dsl))
+#     yaml_string = convert_to_yaml(translated_dsl)
+#     print(yaml_string)
+#     _validate_dsl(_parse_yaml(yaml_string))
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 584320ef..54b03c4e 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1324,14 +1324,15 @@ def test_editor_json_schema_is_valid():
 def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
-    yaml_str = """- tab:
+    yaml_str = """
+- tab:
     en: "counting"
     nl: "tellen"
   contexts:
     - testcases:
       - statement: !natural_language
-          en: 'result = trying(10)'
-          nl: 'resultaat = proberen(10)'
+          en: 'result = Trying(10)'
+          nl: 'resultaat = Proberen(10)'
       - expression: !natural_language
           en: 'count_words(result)'
           nl: 'tel_woorden(resultaat)'
@@ -1354,12 +1355,32 @@ def test_natural_translate_unit_test():
           name: "evaluate_test"
           arguments:
             en: ["The value", "is OK!", "is not OK!"]
-            nl: ["De waarde", "is OK!", "is niet OK!"]    
-    """
-    translated_yaml_str = """- tab: counting
+            nl: ["De waarde", "is OK!", "is niet OK!"]
+        description: !natural_language
+          en: "Ten"
+          nl: "Tien"
+      files:
+        - name: "file.txt"
+          url: "media/workdir/file.txt"
+        - name: "fileNL.txt"
+          url: "media/workdir/fileNL.txt"
+    - testcases:
+        - statement: !natural_language
+            en: 'result = Trying(11)'
+            nl: 'resultaat = Proberen(11)'
+        - expression: 'result'
+          return: '11'
+          description:
+            description:
+              en: "Eleven"
+              nl: "Elf"
+            format: "code"
+""".strip()
+    translated_yaml_str = """
+- tab: counting
   contexts:
   - testcases:
-    - statement: result = trying(10)
+    - statement: result = Trying(10)
     - expression: count_words(result)
       return: The result is 10
     - expression: !expression 'count'
@@ -1374,9 +1395,105 @@ def test_natural_translate_unit_test():
         - The value
         - is OK!
         - is not OK!
-"""
+      description: Ten
+    files:
+    - name: file.txt
+      url: media/workdir/file.txt
+    - name: fileNL.txt
+      url: media/workdir/fileNL.txt
+  - testcases:
+    - statement: result = Trying(11)
+    - expression: result
+      return: '11'
+      description:
+        description: Eleven
+        format: code
+""".strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml.strip() == translated_yaml_str
+
+def test_natural_translate_io_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
+    yaml_str = """
+units:
+  - unit:
+      en: "Arguments"
+      nl: "Argumenten"
+    scripts:
+      - stdin:
+          en: "User"
+          nl: "Gebruiker"
+        arguments:
+          en: [ "input", "output" ]
+          nl: [ "invoer", "uitvoer" ]
+        stdout: !natural_language
+          en: "Hi User"
+          nl: "Hallo Gebruiker"
+        stderr: !natural_language
+          en: "Nothing to see here"
+          nl: "Hier is niets te zien"
+        exception: !natural_language
+          en: "Does not look good"
+          nl: "Ziet er niet goed uit"
+      - stdin:
+          en: "Friend"
+          nl: "Vriend"
+        arguments:
+          en: [ "input", "output" ]
+          nl: [ "invoer", "uitvoer" ]
+        stdout:
+          data:
+            en: "Hi Friend"
+            nl: "Hallo Vriend"
+          config:
+            ignoreWhitespace: true
+        stderr:
+          data:
+            en: "Nothing to see here"
+            nl: "Hier is niets te zien"
+          config:
+            ignoreWhitespace: true
+        exception:
+          message:
+            en: "Does not look good"
+            nl: "Ziet er niet goed uit"
+          types:
+            typescript: "ERROR"
+""".strip()
+    translated_yaml_str = """
+units:
+- unit: Arguments
+  scripts:
+  - stdin: User
+    arguments:
+    - input
+    - output
+    stdout: Hi User
+    stderr: Nothing to see here
+    exception: Does not look good
+  - stdin: Friend
+    arguments:
+    - input
+    - output
+    stdout:
+      data: Hi Friend
+      config:
+        ignoreWhitespace: true
+    stderr:
+      data: Nothing to see here
+      config:
+        ignoreWhitespace: true
+    exception:
+      message: Does not look good
+      types:
+        typescript: ERROR
+""".strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")
     translated_yaml = convert_to_yaml(translated_dsl)
     print(translated_yaml)
-    assert translated_yaml == translated_yaml_str
+    assert translated_yaml.strip() == translated_yaml_str

From e14c7584270c609de64b32ec2efcbabef6682987 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 12 Dec 2024 18:29:50 +0100
Subject: [PATCH 007/111] setup main

---
 tested/nat_translation.py | 24 ++++++++++++------------
 tests/test_dsl_yaml.py    |  6 ++++--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b011496a..ed2aa02b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -202,15 +202,15 @@ def expression_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-# if __name__ == "__main__":
-#     n = len(sys.argv)
-#     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
-#
-#     path = sys.argv[1]
-#     lang = sys.argv[2]
-#     new_yaml = parse_yaml(path)
-#     print(new_yaml)
-#     translated_dsl = translate_dsl(new_yaml, lang)
-#     yaml_string = convert_to_yaml(translated_dsl)
-#     print(yaml_string)
-#     _validate_dsl(_parse_yaml(yaml_string))
+if __name__ == "__main__":
+    n = len(sys.argv)
+    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+
+    path = sys.argv[1]
+    lang = sys.argv[2]
+    new_yaml = parse_yaml(path)
+    print(new_yaml)
+    translated_dsl = translate_dsl(new_yaml, lang)
+    yaml_string = convert_to_yaml(translated_dsl)
+    print(yaml_string)
+    _validate_dsl(_parse_yaml(yaml_string))
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 54b03c4e..9101b16d 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -20,8 +20,8 @@
     StringTypes,
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
-from tested.dsl.translate_parser import load_schema_validator, _parse_yaml
-from tested.nat_translation import translate_dsl, convert_to_yaml
+from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
+from tested.nat_translation import convert_to_yaml, translate_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1321,6 +1321,7 @@ def test_editor_json_schema_is_valid():
     assert isinstance(validator.schema, dict)
     validator.check_schema(validator.schema)
 
+
 def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
@@ -1415,6 +1416,7 @@ def test_natural_translate_unit_test():
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
+
 def test_natural_translate_io_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.

From 65fb097333363fd1499c0f10c5f2398df6bdd418 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 12 Dec 2024 20:06:55 +0100
Subject: [PATCH 008/111] Made a small fix

---
 tested/nat_translation.py | 15 +++++++++------
 tests/test_dsl_yaml.py    | 14 +++++++-------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index ed2aa02b..34f6b28a 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -120,12 +120,15 @@ def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
-        print(f"context: {context}")
-        if "script" in context or "testcases" in context:
-            key_to_set = "script" if "script" in context else "testcases"
-            raw_testcases = context.get(key_to_set)
-            assert isinstance(raw_testcases, list)
-            context[key_to_set] = translate_testcases(raw_testcases, language)
+        key_to_set = "script" if "script" in context else "testcases"
+        raw_testcases = context.get(key_to_set)
+        assert isinstance(raw_testcases, list)
+        context[key_to_set] = translate_testcases(raw_testcases, language)
+        if "files" in context:
+            files = context.get("files")
+            if isinstance(files, NaturalLanguageMap):
+                assert language in files
+                context["files"] = files[language]
         result.append(context)
 
     return result
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 9101b16d..c3ae1f61 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1360,11 +1360,13 @@ def test_natural_translate_unit_test():
         description: !natural_language
           en: "Ten"
           nl: "Tien"
-      files:
-        - name: "file.txt"
-          url: "media/workdir/file.txt"
-        - name: "fileNL.txt"
-          url: "media/workdir/fileNL.txt"
+      files: !natural_language
+        en:
+          - name: "file.txt"
+            url: "media/workdir/file.txt"
+        nl:
+          - name: "fileNL.txt"
+            url: "media/workdir/fileNL.txt"
     - testcases:
         - statement: !natural_language
             en: 'result = Trying(11)'
@@ -1400,8 +1402,6 @@ def test_natural_translate_unit_test():
     files:
     - name: file.txt
       url: media/workdir/file.txt
-    - name: fileNL.txt
-      url: media/workdir/fileNL.txt
   - testcases:
     - statement: result = Trying(11)
     - expression: result

From e941ef6cff0df68f1df6e7d4364b56462d4c6ea3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 13 Dec 2024 12:46:26 +0100
Subject: [PATCH 009/111] Tested an extra edge case

---
 tested/nat_translation.py |   1 +
 tests/test_dsl_yaml.py    | 122 +++++++++++++++++++-------------------
 2 files changed, 63 insertions(+), 60 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 34f6b28a..61ba48db 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -122,6 +122,7 @@ def translate_contexts(contexts: list, language: str) -> list:
         assert isinstance(context, dict)
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
+        print(raw_testcases)
         assert isinstance(raw_testcases, list)
         context[key_to_set] = translate_testcases(raw_testcases, language)
         if "files" in context:
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index c3ae1f61..dbec9bd7 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1425,74 +1425,76 @@ def test_natural_translate_io_test():
   - unit:
       en: "Arguments"
       nl: "Argumenten"
-    scripts:
-      - stdin:
-          en: "User"
-          nl: "Gebruiker"
-        arguments:
-          en: [ "input", "output" ]
-          nl: [ "invoer", "uitvoer" ]
-        stdout: !natural_language
-          en: "Hi User"
-          nl: "Hallo Gebruiker"
-        stderr: !natural_language
-          en: "Nothing to see here"
-          nl: "Hier is niets te zien"
-        exception: !natural_language
-          en: "Does not look good"
-          nl: "Ziet er niet goed uit"
-      - stdin:
-          en: "Friend"
-          nl: "Vriend"
-        arguments:
-          en: [ "input", "output" ]
-          nl: [ "invoer", "uitvoer" ]
-        stdout:
-          data:
-            en: "Hi Friend"
-            nl: "Hallo Vriend"
-          config:
-            ignoreWhitespace: true
-        stderr:
-          data:
+    cases:
+      - script:
+        - stdin:
+            en: "User"
+            nl: "Gebruiker"
+          arguments:
+            en: [ "input", "output" ]
+            nl: [ "invoer", "uitvoer" ]
+          stdout: !natural_language
+            en: "Hi User"
+            nl: "Hallo Gebruiker"
+          stderr: !natural_language
             en: "Nothing to see here"
             nl: "Hier is niets te zien"
-          config:
-            ignoreWhitespace: true
-        exception:
-          message:
+          exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
-          types:
-            typescript: "ERROR"
+        - stdin:
+            en: "Friend"
+            nl: "Vriend"
+          arguments:
+            en: [ "input", "output" ]
+            nl: [ "invoer", "uitvoer" ]
+          stdout:
+            data:
+              en: "Hi Friend"
+              nl: "Hallo Vriend"
+            config:
+              ignoreWhitespace: true
+          stderr:
+            data:
+              en: "Nothing to see here"
+              nl: "Hier is niets te zien"
+            config:
+              ignoreWhitespace: true
+          exception:
+            message:
+              en: "Does not look good"
+              nl: "Ziet er niet goed uit"
+            types:
+              typescript: "ERROR"
 """.strip()
     translated_yaml_str = """
 units:
 - unit: Arguments
-  scripts:
-  - stdin: User
-    arguments:
-    - input
-    - output
-    stdout: Hi User
-    stderr: Nothing to see here
-    exception: Does not look good
-  - stdin: Friend
-    arguments:
-    - input
-    - output
-    stdout:
-      data: Hi Friend
-      config:
-        ignoreWhitespace: true
-    stderr:
-      data: Nothing to see here
-      config:
-        ignoreWhitespace: true
-    exception:
-      message: Does not look good
-      types:
-        typescript: ERROR
+  cases:
+  - script:
+    - stdin: User
+      arguments:
+      - input
+      - output
+      stdout: Hi User
+      stderr: Nothing to see here
+      exception: Does not look good
+    - stdin: Friend
+      arguments:
+      - input
+      - output
+      stdout:
+        data: Hi Friend
+        config:
+          ignoreWhitespace: true
+      stderr:
+        data: Nothing to see here
+        config:
+          ignoreWhitespace: true
+      exception:
+        message: Does not look good
+        types:
+          typescript: ERROR
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")

From eef397b4f7fed9f901d74edd89039585d40e13f7 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 13 Dec 2024 13:45:49 +0100
Subject: [PATCH 010/111] Cleaned up code and added extra cases.

---
 tested/nat_translation.py |  3 ---
 tests/test_dsl_yaml.py    | 21 +++++++++++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 61ba48db..8e8059c6 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -122,7 +122,6 @@ def translate_contexts(contexts: list, language: str) -> list:
         assert isinstance(context, dict)
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
-        print(raw_testcases)
         assert isinstance(raw_testcases, list)
         context[key_to_set] = translate_testcases(raw_testcases, language)
         if "files" in context:
@@ -158,7 +157,6 @@ def translate_tab(tab: YamlDict, language: str) -> YamlDict:
         assert isinstance(tab["testcases"], list)
         tab["testcases"] = translate_testcases(tab["testcases"], language)
     else:
-        print(tab)
         assert "scripts" in tab
         assert isinstance(tab["scripts"], list)
         tab["scripts"] = translate_testcases(tab["scripts"], language)
@@ -213,7 +211,6 @@ def expression_representer(dumper, data):
     path = sys.argv[1]
     lang = sys.argv[2]
     new_yaml = parse_yaml(path)
-    print(new_yaml)
     translated_dsl = translate_dsl(new_yaml, lang)
     yaml_string = convert_to_yaml(translated_dsl)
     print(yaml_string)
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index dbec9bd7..c0a7be5f 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1378,6 +1378,13 @@ def test_natural_translate_unit_test():
               en: "Eleven"
               nl: "Elf"
             format: "code"
+- tab: 'test'
+  testcases:
+    - expression: !natural_language
+        en: "tests(11)"
+        nl: "testen(11)"
+      return: 11
+
 """.strip()
     translated_yaml_str = """
 - tab: counting
@@ -1409,6 +1416,10 @@ def test_natural_translate_unit_test():
       description:
         description: Eleven
         format: code
+- tab: test
+  testcases:
+  - expression: tests(11)
+    return: 11
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")
@@ -1466,6 +1477,12 @@ def test_natural_translate_io_test():
               nl: "Ziet er niet goed uit"
             types:
               typescript: "ERROR"
+  - unit: "test"
+    scripts:
+      - expression: !natural_language
+            en: "tests(11)"
+            nl: "testen(11)"
+        return: 11
 """.strip()
     translated_yaml_str = """
 units:
@@ -1495,6 +1512,10 @@ def test_natural_translate_io_test():
         message: Does not look good
         types:
           typescript: ERROR
+- unit: test
+  scripts:
+  - expression: tests(11)
+    return: 11
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")

From 30bcdccb380b7b41c89519720d92ca27c15325dc Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 13 Dec 2024 19:04:20 +0100
Subject: [PATCH 011/111] Started on usage with translation table.

---
 tested/nat_translation.py | 100 +++++++++++++++++++++++++++++++++-----
 1 file changed, 87 insertions(+), 13 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 8e8059c6..a3a8a85e 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,6 @@
+import re
 import sys
+from typing import cast
 
 import yaml
 
@@ -14,8 +16,35 @@
 )
 
 
-def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
+def get_replacement(language: str, translation_stack: list, match: re.Match) -> str:
+    word = match.group(1)
+    current = -1
+    stack = translation_stack[current]
+    while abs(current) <= len(translation_stack) and word not in stack:
+        current -= 1
+        stack = translation_stack[current]
+    if abs(current) <= len(translation_stack):
+        translations = stack[word]
+        assert language in translations
+        word = translations[language]
+
+    return word
+
+def flatten_stack(translation_stack: list) -> dict:
+    flattened = {}
+    for d in reversed(translation_stack):
+        flattened.update(d)
+    return flattened
+
+def format_string(string: str, flattened) -> str:
+    return string.format(**flattened)
+
+
+def translate_testcase(
+    testcase: YamlDict, language: str, translation_stack: list
+) -> YamlDict:
     _validate_testcase_combinations(testcase)
+    flat_stack = flatten_stack(translation_stack)
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
@@ -23,17 +52,33 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(expr_stmt, NaturalLanguageMap):
             assert language in expr_stmt
             testcase[key_to_set] = expr_stmt[language]
+
+        # Perform translation based of translation stack.
+        expr_stmt = testcase.get(key_to_set)
+        if isinstance(expr_stmt, dict):
+            testcase[key_to_set] = {
+                k: format_string(cast(str, v), flat_stack) for k, v in expr_stmt.items()
+            }
+        elif isinstance(expr_stmt, str):
+            testcase[key_to_set] = format_string(expr_stmt, flat_stack)
+
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
             if isinstance(stdin_stmt, dict):
                 assert language in stdin_stmt
                 testcase["stdin"] = stdin_stmt[language]
 
+            # Perform translation based of translation stack.
+            testcase["stdin"] = format_string(testcase["stdin"], flat_stack)
+
         arguments = testcase.get("arguments", [])
         if isinstance(arguments, dict):
             assert language in arguments
             testcase["arguments"] = arguments[language]
 
+        # Perform translation based of translation stack.
+        testcase["arguments"] = format_string(testcase["arguments"], flat_stack)
+
     if (stdout := testcase.get("stdout")) is not None:
         # Must use !natural_language
         if isinstance(stdout, NaturalLanguageMap):
@@ -107,34 +152,43 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
     return testcase
 
 
-def translate_testcases(testcases: list, language: str) -> list:
+def translate_testcases(
+    testcases: list, language: str, translation_stack: list
+) -> list:
     result = []
     for testcase in testcases:
         assert isinstance(testcase, dict)
-        result.append(translate_testcase(testcase, language))
+        result.append(translate_testcase(testcase, language, translation_stack))
 
     return result
 
 
-def translate_contexts(contexts: list, language: str) -> list:
+def translate_contexts(contexts: list, language: str, translation_stack: list) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
+        if "translation" in context:
+            translation_stack.append(context["translation"])
+            context.pop("translation")
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
-        context[key_to_set] = translate_testcases(raw_testcases, language)
+        context[key_to_set] = translate_testcases(
+            raw_testcases, language, translation_stack
+        )
         if "files" in context:
             files = context.get("files")
             if isinstance(files, NaturalLanguageMap):
                 assert language in files
                 context["files"] = files[language]
         result.append(context)
+        if "translation" in context:
+            translation_stack.pop()
 
     return result
 
 
-def translate_tab(tab: YamlDict, language: str) -> YamlDict:
+def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> YamlDict:
     key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
 
@@ -142,32 +196,48 @@ def translate_tab(tab: YamlDict, language: str) -> YamlDict:
         assert language in name
         tab[key_to_set] = name[language]
 
+    tab[key_to_set] = format_string(tab[key_to_set], flatten_stack(translation_stack))
+
     # The tab can have testcases or contexts.
     if "contexts" in tab:
         assert isinstance(tab["contexts"], list)
-        tab["contexts"] = translate_contexts(tab["contexts"], language)
+        tab["contexts"] = translate_contexts(
+            tab["contexts"], language, translation_stack
+        )
     elif "cases" in tab:
         assert "unit" in tab
         # We have testcases N.S. / contexts O.S.
         assert isinstance(tab["cases"], list)
-        tab["cases"] = translate_contexts(tab["cases"], language)
+        tab["cases"] = translate_contexts(tab["cases"], language, translation_stack)
     elif "testcases" in tab:
         # We have scripts N.S. / testcases O.S.
         assert "tab" in tab
         assert isinstance(tab["testcases"], list)
-        tab["testcases"] = translate_testcases(tab["testcases"], language)
+        tab["testcases"] = translate_testcases(
+            tab["testcases"], language, translation_stack
+        )
     else:
         assert "scripts" in tab
         assert isinstance(tab["scripts"], list)
-        tab["scripts"] = translate_testcases(tab["scripts"], language)
+        tab["scripts"] = translate_testcases(
+            tab["scripts"], language, translation_stack
+        )
     return tab
 
 
-def translate_tabs(dsl_list: list, language: str) -> list:
+def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> list:
+    if translation_stack is None:
+        translation_stack = []
+
     result = []
     for tab in dsl_list:
         assert isinstance(tab, dict)
-        result.append(translate_tab(tab, language))
+        if "translation" in tab:
+            translation_stack.append(tab["translation"])
+            tab.pop("translation")
+        result.append(translate_tab(tab, language, translation_stack))
+        if "translation" in tab:
+            translation_stack.pop()
 
     return result
 
@@ -180,7 +250,11 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
         key_to_set = "units" if "units" in dsl_object else "tabs"
         tab_list = dsl_object.get(key_to_set)
         assert isinstance(tab_list, list)
-        dsl_object[key_to_set] = translate_tabs(tab_list, language)
+        translation_stack = []
+        if "translation" in dsl_object:
+            translation_stack.append(dsl_object["translation"])
+            dsl_object.pop("translation")
+        dsl_object[key_to_set] = translate_tabs(tab_list, language, translation_stack)
         return dsl_object
 
 

From 4230003b370b91b9843f96abbe74143faf7683a0 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 14 Dec 2024 20:29:29 +0100
Subject: [PATCH 012/111] Added support for translation-table in global scope,
 tab-scope and context-scope

---
 tested/nat_translation.py | 79 +++++++++++++++++++++++++++++++++++----
 1 file changed, 71 insertions(+), 8 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index a3a8a85e..9bff4ed3 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,6 +16,17 @@
 )
 
 
+def parse_value(value: list | str | int | float | dict, flattened_stack: dict):
+    if isinstance(value, str):
+        return format_string(value, flattened_stack)
+    elif isinstance(value, dict):
+        return {k: parse_value(v, flattened_stack) for k, v in value.items()}
+    elif isinstance(value, list):
+        return [parse_value(v, flattened_stack) for v in value]
+
+    return value
+
+
 def get_replacement(language: str, translation_stack: list, match: re.Match) -> str:
     word = match.group(1)
     current = -1
@@ -30,12 +41,16 @@ def get_replacement(language: str, translation_stack: list, match: re.Match) ->
 
     return word
 
-def flatten_stack(translation_stack: list) -> dict:
+
+def flatten_stack(translation_stack: list, language: str) -> dict:
     flattened = {}
-    for d in reversed(translation_stack):
-        flattened.update(d)
+    for d in translation_stack:
+
+        flattened.update({k: v[language] for k, v in d.items() if language in v})
+    print(f"flattened: {flattened}")
     return flattened
 
+
 def format_string(string: str, flattened) -> str:
     return string.format(**flattened)
 
@@ -44,7 +59,7 @@ def translate_testcase(
     testcase: YamlDict, language: str, translation_stack: list
 ) -> YamlDict:
     _validate_testcase_combinations(testcase)
-    flat_stack = flatten_stack(translation_stack)
+    flat_stack = flatten_stack(translation_stack, language)
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
@@ -77,7 +92,9 @@ def translate_testcase(
             testcase["arguments"] = arguments[language]
 
         # Perform translation based of translation stack.
-        testcase["arguments"] = format_string(testcase["arguments"], flat_stack)
+        testcase["arguments"] = [
+            format_string(arg, flat_stack) for arg in testcase["arguments"]
+        ]
 
     if (stdout := testcase.get("stdout")) is not None:
         # Must use !natural_language
@@ -90,11 +107,20 @@ def translate_testcase(
                 assert language in data
                 stdout["data"] = data[language]
                 testcase["stdout"] = stdout
+
+        # Perform translation based of translation stack.
+        stdout = testcase.get("stdout")
+        if isinstance(stdout, dict):
+            testcase["stdout"]["data"] = format_string(stdout["data"], flat_stack)
+        elif isinstance(stdout, str):
+            testcase["stdout"] = format_string(stdout, flat_stack)
+
     if (file := testcase.get("file")) is not None:
         # Must use !natural_language
         if isinstance(file, NaturalLanguageMap):
             assert language in file
             testcase["file"] = file[language]
+        # TODO: SHOULD I ADD SUPPORT FOR TRANSLATION STACK HERE?
     if (stderr := testcase.get("stderr")) is not None:
         # Must use !natural_language
         if isinstance(stderr, NaturalLanguageMap):
@@ -107,6 +133,13 @@ def translate_testcase(
                 stderr["data"] = data[language]
                 testcase["stderr"] = stderr
 
+        # Perform translation based of translation stack.
+        stderr = testcase.get("stderr")
+        if isinstance(stderr, dict):
+            testcase["stderr"]["data"] = format_string(stderr["data"], flat_stack)
+        elif isinstance(stderr, str):
+            testcase["stderr"] = format_string(stderr, flat_stack)
+
     if (exception := testcase.get("exception")) is not None:
         if isinstance(exception, NaturalLanguageMap):
             assert language in exception
@@ -118,6 +151,15 @@ def translate_testcase(
                 exception["message"] = message[language]
                 testcase["exception"] = exception
 
+        # Perform translation based of translation stack.
+        exception = testcase.get("exception")
+        if isinstance(stderr, dict):
+            testcase["exception"]["message"] = format_string(
+                exception["message"], flat_stack
+            )
+        elif isinstance(stderr, str):
+            testcase["exception"] = format_string(exception, flat_stack)
+
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
@@ -125,30 +167,46 @@ def translate_testcase(
                 assert language in arguments
                 result["arguments"] = arguments[language]
 
+            # Perform translation based of translation stack.
+            result["arguments"] = [
+                format_string(arg, flat_stack) for arg in result["arguments"]
+            ]
+
             value = result.get("value")
             # Must use !natural_language
             if isinstance(value, NaturalLanguageMap):
                 assert language in value
                 result["value"] = value[language]
 
+            result["value"] = parse_value(result["value"], flat_stack)
+
             testcase["return"] = result
         elif isinstance(result, NaturalLanguageMap):
             # Must use !natural_language
             assert language in result
             testcase["return"] = result[language]
 
+        testcase["return"] = parse_value(testcase["return"], flat_stack)
+
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
         if isinstance(description, NaturalLanguageMap):
             assert language in description
             testcase["description"] = description[language]
-        elif isinstance(description, dict):
+
+        if isinstance(testcase["description"], dict):
             dd = description["description"]
             if isinstance(dd, dict):
                 assert language in dd
                 description["description"] = dd[language]
                 testcase["description"] = description
 
+            if isinstance(description["description"], str):
+                description["description"] = format_string(
+                    description["description"], flat_stack
+                )
+                testcase["description"] = description
+
     return testcase
 
 
@@ -196,7 +254,9 @@ def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> Yaml
         assert language in name
         tab[key_to_set] = name[language]
 
-    tab[key_to_set] = format_string(tab[key_to_set], flatten_stack(translation_stack))
+    tab[key_to_set] = format_string(
+        tab[key_to_set], flatten_stack(translation_stack, language)
+    )
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:
@@ -232,12 +292,15 @@ def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> lis
     result = []
     for tab in dsl_list:
         assert isinstance(tab, dict)
+
         if "translation" in tab:
             translation_stack.append(tab["translation"])
-            tab.pop("translation")
+        print(f"tab : {translation_stack}")
+
         result.append(translate_tab(tab, language, translation_stack))
         if "translation" in tab:
             translation_stack.pop()
+            tab.pop("translation")
 
     return result
 

From 1ddef15cc5c2e24ab37b7b8d2b502783be5f65aa Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 15 Dec 2024 17:51:17 +0100
Subject: [PATCH 013/111] Cleaned up code and fixed pyright issue

---
 tested/nat_translation.py | 123 ++++++++++++++------------------------
 1 file changed, 46 insertions(+), 77 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 9bff4ed3..bbefc255 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -55,6 +55,25 @@ def format_string(string: str, flattened) -> str:
     return string.format(**flattened)
 
 
+def translate_io(
+    io_object: YamlObject, key: str, language: str, flat_stack: dict
+) -> str:
+    if isinstance(io_object, NaturalLanguageMap):
+        assert language in io_object
+        io_object = io_object[language]
+    elif isinstance(io_object, dict):
+        data = io_object[key]
+        if isinstance(data, dict):
+            assert language in data
+            data = data[language]
+            assert isinstance(data, str)
+            io_object[key] = format_string(data, flat_stack)
+
+    # Perform translation based of translation stack.
+    assert isinstance(io_object, str)
+    return format_string(io_object, flat_stack)
+
+
 def translate_testcase(
     testcase: YamlDict, language: str, translation_stack: list
 ) -> YamlDict:
@@ -66,10 +85,9 @@ def translate_testcase(
         # Must use !natural_language
         if isinstance(expr_stmt, NaturalLanguageMap):
             assert language in expr_stmt
-            testcase[key_to_set] = expr_stmt[language]
+            expr_stmt = expr_stmt[language]
 
         # Perform translation based of translation stack.
-        expr_stmt = testcase.get(key_to_set)
         if isinstance(expr_stmt, dict):
             testcase[key_to_set] = {
                 k: format_string(cast(str, v), flat_stack) for k, v in expr_stmt.items()
@@ -81,39 +99,26 @@ def translate_testcase(
         if (stdin_stmt := testcase.get("stdin")) is not None:
             if isinstance(stdin_stmt, dict):
                 assert language in stdin_stmt
-                testcase["stdin"] = stdin_stmt[language]
+                stdin_stmt = stdin_stmt[language]
 
             # Perform translation based of translation stack.
-            testcase["stdin"] = format_string(testcase["stdin"], flat_stack)
+            assert isinstance(stdin_stmt, str)
+            testcase["stdin"] = format_string(stdin_stmt, flat_stack)
 
         arguments = testcase.get("arguments", [])
         if isinstance(arguments, dict):
             assert language in arguments
-            testcase["arguments"] = arguments[language]
+            arguments = arguments[language]
 
         # Perform translation based of translation stack.
+        assert isinstance(arguments, list)
         testcase["arguments"] = [
-            format_string(arg, flat_stack) for arg in testcase["arguments"]
+            format_string(str(arg), flat_stack) for arg in arguments
         ]
 
     if (stdout := testcase.get("stdout")) is not None:
         # Must use !natural_language
-        if isinstance(stdout, NaturalLanguageMap):
-            assert language in stdout
-            testcase["stdout"] = stdout[language]
-        elif isinstance(stdout, dict):
-            data = stdout["data"]
-            if isinstance(data, dict):
-                assert language in data
-                stdout["data"] = data[language]
-                testcase["stdout"] = stdout
-
-        # Perform translation based of translation stack.
-        stdout = testcase.get("stdout")
-        if isinstance(stdout, dict):
-            testcase["stdout"]["data"] = format_string(stdout["data"], flat_stack)
-        elif isinstance(stdout, str):
-            testcase["stdout"] = format_string(stdout, flat_stack)
+        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack)
 
     if (file := testcase.get("file")) is not None:
         # Must use !natural_language
@@ -122,90 +127,55 @@ def translate_testcase(
             testcase["file"] = file[language]
         # TODO: SHOULD I ADD SUPPORT FOR TRANSLATION STACK HERE?
     if (stderr := testcase.get("stderr")) is not None:
-        # Must use !natural_language
-        if isinstance(stderr, NaturalLanguageMap):
-            assert language in stderr
-            testcase["stderr"] = stderr[language]
-        elif isinstance(stderr, dict):
-            data = stderr["data"]
-            if isinstance(data, dict):
-                assert language in data
-                stderr["data"] = data[language]
-                testcase["stderr"] = stderr
-
-        # Perform translation based of translation stack.
-        stderr = testcase.get("stderr")
-        if isinstance(stderr, dict):
-            testcase["stderr"]["data"] = format_string(stderr["data"], flat_stack)
-        elif isinstance(stderr, str):
-            testcase["stderr"] = format_string(stderr, flat_stack)
+        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack)
 
     if (exception := testcase.get("exception")) is not None:
-        if isinstance(exception, NaturalLanguageMap):
-            assert language in exception
-            testcase["exception"] = exception[language]
-        elif isinstance(exception, dict):
-            message = exception["message"]
-            if isinstance(message, dict):
-                assert language in message
-                exception["message"] = message[language]
-                testcase["exception"] = exception
-
-        # Perform translation based of translation stack.
-        exception = testcase.get("exception")
-        if isinstance(stderr, dict):
-            testcase["exception"]["message"] = format_string(
-                exception["message"], flat_stack
-            )
-        elif isinstance(stderr, str):
-            testcase["exception"] = format_string(exception, flat_stack)
+        testcase["exception"] = translate_io(exception, "message", language, flat_stack)
 
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
             if isinstance(arguments, dict):
                 assert language in arguments
-                result["arguments"] = arguments[language]
+                arguments = arguments[language]
 
             # Perform translation based of translation stack.
             result["arguments"] = [
-                format_string(arg, flat_stack) for arg in result["arguments"]
+                format_string(str(arg), flat_stack) for arg in arguments
             ]
 
             value = result.get("value")
             # Must use !natural_language
             if isinstance(value, NaturalLanguageMap):
                 assert language in value
-                result["value"] = value[language]
+                value = value[language]
 
-            result["value"] = parse_value(result["value"], flat_stack)
+            assert isinstance(value, str)
+            result["value"] = parse_value(value, flat_stack)
 
-            testcase["return"] = result
         elif isinstance(result, NaturalLanguageMap):
             # Must use !natural_language
             assert language in result
-            testcase["return"] = result[language]
+            result = result[language]
 
-        testcase["return"] = parse_value(testcase["return"], flat_stack)
+        testcase["return"] = parse_value(result, flat_stack)
 
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
         if isinstance(description, NaturalLanguageMap):
             assert language in description
-            testcase["description"] = description[language]
+            description = description[language]
 
-        if isinstance(testcase["description"], dict):
+        if isinstance(description, dict):
             dd = description["description"]
             if isinstance(dd, dict):
                 assert language in dd
-                description["description"] = dd[language]
-                testcase["description"] = description
+                dd = dd[language]
+
+            if isinstance(dd, str):
+                description["description"] = format_string(dd, flat_stack)
 
-            if isinstance(description["description"], str):
-                description["description"] = format_string(
-                    description["description"], flat_stack
-                )
-                testcase["description"] = description
+        testcase["description"] = description
 
     return testcase
 
@@ -252,11 +222,10 @@ def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> Yaml
 
     if isinstance(name, dict):
         assert language in name
-        tab[key_to_set] = name[language]
+        name = name[language]
 
-    tab[key_to_set] = format_string(
-        tab[key_to_set], flatten_stack(translation_stack, language)
-    )
+    assert isinstance(name, str)
+    tab[key_to_set] = format_string(name, flatten_stack(translation_stack, language))
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:

From 5dabc80f6d18332b476d75554789ad882f3486b3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 15 Dec 2024 19:21:55 +0100
Subject: [PATCH 014/111] fixed tests and added more

---
 tested/nat_translation.py |  21 ++--
 tests/test_dsl_yaml.py    | 201 +++++++++++++++++++++-----------------
 2 files changed, 124 insertions(+), 98 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index bbefc255..8dc32424 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,6 +1,6 @@
 import re
 import sys
-from typing import cast
+from typing import cast, Any
 
 import yaml
 
@@ -47,7 +47,6 @@ def flatten_stack(translation_stack: list, language: str) -> dict:
     for d in translation_stack:
 
         flattened.update({k: v[language] for k, v in d.items() if language in v})
-    print(f"flattened: {flattened}")
     return flattened
 
 
@@ -57,7 +56,7 @@ def format_string(string: str, flattened) -> str:
 
 def translate_io(
     io_object: YamlObject, key: str, language: str, flat_stack: dict
-) -> str:
+) -> str | dict:
     if isinstance(io_object, NaturalLanguageMap):
         assert language in io_object
         io_object = io_object[language]
@@ -70,8 +69,11 @@ def translate_io(
             io_object[key] = format_string(data, flat_stack)
 
     # Perform translation based of translation stack.
-    assert isinstance(io_object, str)
-    return format_string(io_object, flat_stack)
+    print(io_object)
+    if isinstance(io_object, str):
+        return format_string(io_object, flat_stack)
+
+    return io_object
 
 
 def translate_testcase(
@@ -158,7 +160,10 @@ def translate_testcase(
             assert language in result
             result = result[language]
 
-        testcase["return"] = parse_value(result, flat_stack)
+        if isinstance(result, str):
+            result = parse_value(result, flat_stack)
+
+        testcase["return"] = result
 
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
@@ -197,7 +202,7 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
         assert isinstance(context, dict)
         if "translation" in context:
             translation_stack.append(context["translation"])
-            context.pop("translation")
+
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
@@ -212,6 +217,7 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
         result.append(context)
         if "translation" in context:
             translation_stack.pop()
+            context.pop("translation")
 
     return result
 
@@ -264,7 +270,6 @@ def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> lis
 
         if "translation" in tab:
             translation_stack.append(tab["translation"])
-        print(f"tab : {translation_stack}")
 
         result.append(translate_tab(tab, language, translation_stack))
         if "translation" in tab:
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index c0a7be5f..1d93f4ac 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1326,78 +1326,95 @@ def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
     yaml_str = """
-- tab:
-    en: "counting"
-    nl: "tellen"
-  contexts:
-    - testcases:
-      - statement: !natural_language
-          en: 'result = Trying(10)'
-          nl: 'resultaat = Proberen(10)'
-      - expression: !natural_language
-          en: 'count_words(result)'
-          nl: 'tel_woorden(resultaat)'
-        return: !natural_language
-          en: 'The result is 10'
-          nl: 'Het resultaat is 10'
-      - expression: !natural_language
-          en: !expression "count"
-          nl: !expression "tellen"
-        return: !natural_language
-          en: 'count'
-          nl: 'tellen'
-      - expression: 'ok(10)'
-        return: !oracle
-          value: !natural_language
-            en: "The value 10 is OK!"
-            nl: "De waarde 10 is OK!"
-          oracle: "custom_check"
-          file: "test.py"
-          name: "evaluate_test"
-          arguments:
-            en: ["The value", "is OK!", "is not OK!"]
-            nl: ["De waarde", "is OK!", "is niet OK!"]
-        description: !natural_language
-          en: "Ten"
-          nl: "Tien"
-      files: !natural_language
-        en:
-          - name: "file.txt"
-            url: "media/workdir/file.txt"
-        nl:
-          - name: "fileNL.txt"
-            url: "media/workdir/fileNL.txt"
-    - testcases:
+translation:
+  animal:
+    en: "animals"
+    nl: "dieren"
+  result:
+    en: "results"
+    nl: "resultaten"
+  elf:
+    en: "eleven"
+    nl: "elf"
+tabs:
+  - tab: "{{{animal}}}_{{{result}}}"
+    translation:
+      animal:
+        en: "animal_tab"
+        nl: "dier_tab"
+    contexts:
+      - testcases:
         - statement: !natural_language
-            en: 'result = Trying(11)'
-            nl: 'resultaat = Proberen(11)'
-        - expression: 'result'
-          return: '11'
-          description:
+            en: '{result} = Trying(10)'
+            nl: '{result} = Proberen(10)'
+        - expression: !natural_language
+            en: 'count_words({result})'
+            nl: 'tel_woorden({result})'
+          return: !natural_language
+            en: 'The {result} is 10'
+            nl: 'Het {result} is 10'
+        - expression: !natural_language
+            en: !expression "count"
+            nl: !expression "tellen"
+          return: !natural_language
+            en: 'count'
+            nl: 'tellen'
+        - expression: 'ok(10)'
+          return: !oracle
+            value: !natural_language
+              en: "The {result} 10 is OK!"
+              nl: "Het {result} 10 is OK!"
+            oracle: "custom_check"
+            file: "test.py"
+            name: "evaluate_test"
+            arguments:
+              en: ["The value", "is OK!", "is not OK!"]
+              nl: ["Het {result}", "is OK!", "is niet OK!"]
+          description: !natural_language
+            en: "Ten"
+            nl: "Tien"
+        files: !natural_language
+          en:
+            - name: "file.txt"
+              url: "media/workdir/file.txt"
+          nl:
+            - name: "fileNL.txt"
+              url: "media/workdir/fileNL.txt"
+        translation:
+          result:
+            en: "results_context"
+            nl: "resultaten_context"
+      - testcases:
+          - statement: !natural_language
+              en: 'result = Trying(11)'
+              nl: 'resultaat = Proberen(11)'
+          - expression: 'result'
+            return: '11_{elf}'
             description:
-              en: "Eleven"
-              nl: "Elf"
-            format: "code"
-- tab: 'test'
-  testcases:
-    - expression: !natural_language
-        en: "tests(11)"
-        nl: "testen(11)"
-      return: 11
-
+              description:
+                en: "Eleven_{elf}"
+                nl: "Elf_{elf}"
+              format: "code"
+  - tab: '{animal}'
+    testcases:
+      - expression: !natural_language
+          en: "tests(11)"
+          nl: "testen(11)"
+        return: 11
 """.strip()
     translated_yaml_str = """
-- tab: counting
+tabs:
+- tab: '{animal_tab}_{results}'
   contexts:
   - testcases:
-    - statement: result = Trying(10)
-    - expression: count_words(result)
-      return: The result is 10
+    - statement: results_context = Trying(10)
+    - expression: count_words(results_context)
+      return: The results_context is 10
     - expression: !expression 'count'
       return: count
     - expression: ok(10)
       return: !oracle
-        value: The value 10 is OK!
+        value: The results_context 10 is OK!
         oracle: custom_check
         file: test.py
         name: evaluate_test
@@ -1412,11 +1429,11 @@ def test_natural_translate_unit_test():
   - testcases:
     - statement: result = Trying(11)
     - expression: result
-      return: '11'
+      return: 11_eleven
       description:
-        description: Eleven
+        description: Eleven_eleven
         format: code
-- tab: test
+- tab: animals
   testcases:
   - expression: tests(11)
     return: 11
@@ -1436,45 +1453,49 @@ def test_natural_translate_io_test():
   - unit:
       en: "Arguments"
       nl: "Argumenten"
+    translation:
+      User:
+        en: "user"
+        nl: "gebruiker"
     cases:
       - script:
         - stdin:
-            en: "User"
-            nl: "Gebruiker"
+            en: "User_{User}"
+            nl: "Gebruiker_{User}"
           arguments:
-            en: [ "input", "output" ]
-            nl: [ "invoer", "uitvoer" ]
+            en: [ "input_{User}", "output_{User}" ]
+            nl: [ "invoer_{User}", "uitvoer_{User}" ]
           stdout: !natural_language
-            en: "Hi User"
-            nl: "Hallo Gebruiker"
+            en: "Hi {User}"
+            nl: "Hallo {User}"
           stderr: !natural_language
-            en: "Nothing to see here"
-            nl: "Hier is niets te zien"
+            en: "Nothing to see here {User}"
+            nl: "Hier is niets te zien {User}"
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
         - stdin:
-            en: "Friend"
-            nl: "Vriend"
+            en: "Friend of {User}"
+            nl: "Vriend van {User}"
           arguments:
             en: [ "input", "output" ]
             nl: [ "invoer", "uitvoer" ]
           stdout:
             data:
-              en: "Hi Friend"
-              nl: "Hallo Vriend"
+              en: "Hi Friend of {User}"
+              nl: "Hallo Vriend van {User}"
             config:
               ignoreWhitespace: true
           stderr:
             data:
-              en: "Nothing to see here"
-              nl: "Hier is niets te zien"
+              en: "Nothing to see here {User}"
+              nl: "Hier is niets te zien {User}"
             config:
               ignoreWhitespace: true
           exception:
             message:
-              en: "Does not look good"
-              nl: "Ziet er niet goed uit"
+              en: "Does not look good {User}"
+              nl: "Ziet er niet goed uit {User}"
             types:
               typescript: "ERROR"
   - unit: "test"
@@ -1489,27 +1510,27 @@ def test_natural_translate_io_test():
 - unit: Arguments
   cases:
   - script:
-    - stdin: User
+    - stdin: User_user
       arguments:
-      - input
-      - output
-      stdout: Hi User
-      stderr: Nothing to see here
+      - input_user
+      - output_user
+      stdout: Hi user
+      stderr: Nothing to see here user
       exception: Does not look good
-    - stdin: Friend
+    - stdin: Friend of user
       arguments:
       - input
       - output
       stdout:
-        data: Hi Friend
+        data: Hi Friend of user
         config:
           ignoreWhitespace: true
       stderr:
-        data: Nothing to see here
+        data: Nothing to see here user
         config:
           ignoreWhitespace: true
       exception:
-        message: Does not look good
+        message: Does not look good user
         types:
           typescript: ERROR
 - unit: test

From 69a77d30edada68c9b33ea11b976266c774dbf09 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 15 Dec 2024 19:29:15 +0100
Subject: [PATCH 015/111] fixed some small issues

---
 tested/nat_translation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 8dc32424..1b6edfec 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,6 +1,6 @@
 import re
 import sys
-from typing import cast, Any
+from typing import cast
 
 import yaml
 
@@ -56,7 +56,7 @@ def format_string(string: str, flattened) -> str:
 
 def translate_io(
     io_object: YamlObject, key: str, language: str, flat_stack: dict
-) -> str | dict:
+) -> YamlObject:
     if isinstance(io_object, NaturalLanguageMap):
         assert language in io_object
         io_object = io_object[language]

From eb62f92b4e412ccba82ac97b06b7c0302ddcf0a3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 11:40:50 +0100
Subject: [PATCH 016/111] made some small fixes

---
 tested/nat_translation.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 1b6edfec..0b938bd3 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -60,13 +60,13 @@ def translate_io(
     if isinstance(io_object, NaturalLanguageMap):
         assert language in io_object
         io_object = io_object[language]
-    elif isinstance(io_object, dict):
+    if isinstance(io_object, dict):
         data = io_object[key]
         if isinstance(data, dict):
             assert language in data
             data = data[language]
-            assert isinstance(data, str)
-            io_object[key] = format_string(data, flat_stack)
+        assert isinstance(data, str)
+        io_object[key] = format_string(data, flat_stack)
 
     # Perform translation based of translation stack.
     print(io_object)
@@ -154,16 +154,14 @@ def translate_testcase(
 
             assert isinstance(value, str)
             result["value"] = parse_value(value, flat_stack)
+            testcase["return"] = result
 
         elif isinstance(result, NaturalLanguageMap):
             # Must use !natural_language
             assert language in result
-            result = result[language]
-
-        if isinstance(result, str):
-            result = parse_value(result, flat_stack)
-
-        testcase["return"] = result
+            testcase["return"] = parse_value(result[language], flat_stack)
+        elif result is not None:
+            testcase["return"] = parse_value(result, flat_stack)
 
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
@@ -171,14 +169,17 @@ def translate_testcase(
             assert language in description
             description = description[language]
 
+        if isinstance(description, str):
+            testcase["description"] = format_string(description, flat_stack)
+
         if isinstance(description, dict):
             dd = description["description"]
             if isinstance(dd, dict):
                 assert language in dd
                 dd = dd[language]
 
-            if isinstance(dd, str):
-                description["description"] = format_string(dd, flat_stack)
+            assert isinstance(dd, str)
+            description["description"] = format_string(dd, flat_stack)
 
         testcase["description"] = description
 

From 6e258cf9fb67cce268c4ec396696b3bc84b42960 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 12:47:35 +0100
Subject: [PATCH 017/111] wrote an extra test

---
 tested/nat_translation.py | 2 +-
 tests/test_dsl_yaml.py    | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 0b938bd3..4b483211 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,7 +16,7 @@
 )
 
 
-def parse_value(value: list | str | int | float | dict, flattened_stack: dict):
+def parse_value(value: list | str | int | float | dict, flattened_stack: dict) -> list | str | int | float | dict:
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 1d93f4ac..de451136 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,7 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, translate_dsl
+from tested.nat_translation import convert_to_yaml, translate_dsl, parse_value
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1543,3 +1543,10 @@ def test_natural_translate_io_test():
     translated_yaml = convert_to_yaml(translated_dsl)
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
+
+def test_translate_parse():
+    flattend_stack = {"animal": "dier", "human": "mens", "number": "getal"}
+    value = {"key1": ["value1_{animal}", "value1_{human}"], "key2": "value2_{number}", "key3": 10}
+    expected_value = {"key1": ["value1_dier", "value1_mens"], "key2": "value2_getal", "key3": 10}
+    parsed_result = parse_value(value, flattend_stack)
+    assert parsed_result == expected_value

From 1db1db2355af67d92b7b3e62a3efb0e2a0205b4b Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 12:51:57 +0100
Subject: [PATCH 018/111] fix spelling mistake

---
 tests/test_dsl_yaml.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index de451136..decc20f9 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1545,8 +1545,8 @@ def test_natural_translate_io_test():
     assert translated_yaml.strip() == translated_yaml_str
 
 def test_translate_parse():
-    flattend_stack = {"animal": "dier", "human": "mens", "number": "getal"}
+    flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
     value = {"key1": ["value1_{animal}", "value1_{human}"], "key2": "value2_{number}", "key3": 10}
     expected_value = {"key1": ["value1_dier", "value1_mens"], "key2": "value2_getal", "key3": 10}
-    parsed_result = parse_value(value, flattend_stack)
+    parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value

From 4dedd8c34327d4262a85f9937841f58c9d835564 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 13:00:55 +0100
Subject: [PATCH 019/111] fixed linting issue

---
 tested/nat_translation.py |  4 +++-
 tests/test_dsl_yaml.py    | 13 +++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 4b483211..da3bc60d 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,7 +16,9 @@
 )
 
 
-def parse_value(value: list | str | int | float | dict, flattened_stack: dict) -> list | str | int | float | dict:
+def parse_value(
+    value: list | str | int | float | dict, flattened_stack: dict
+) -> list | str | int | float | dict:
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index decc20f9..fe597364 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1544,9 +1544,18 @@ def test_natural_translate_io_test():
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
+
 def test_translate_parse():
     flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
-    value = {"key1": ["value1_{animal}", "value1_{human}"], "key2": "value2_{number}", "key3": 10}
-    expected_value = {"key1": ["value1_dier", "value1_mens"], "key2": "value2_getal", "key3": 10}
+    value = {
+        "key1": ["value1_{animal}", "value1_{human}"],
+        "key2": "value2_{number}",
+        "key3": 10,
+    }
+    expected_value = {
+        "key1": ["value1_dier", "value1_mens"],
+        "key2": "value2_getal",
+        "key3": 10,
+    }
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value

From b9786c2644ae1f4929f0a87e124bfdde7fd024d1 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 17:43:08 +0100
Subject: [PATCH 020/111] increasing test coverage

---
 tests/test_dsl_yaml.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index fe597364..87900fae 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,7 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, translate_dsl, parse_value
+from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1401,6 +1401,12 @@ def test_natural_translate_unit_test():
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
+      - expression:
+          javascript: "{animal}_javascript(1 + 1)"
+          typescript: "{animal}_typescript(1 + 1)"
+          java: "Submission.{animal}_java(1 + 1)"
+          python: "{animal}_python(1 + 1)"
+        return: 2
 """.strip()
     translated_yaml_str = """
 tabs:
@@ -1437,6 +1443,12 @@ def test_natural_translate_unit_test():
   testcases:
   - expression: tests(11)
     return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python: animals_python(1 + 1)
+    return: 2
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")

From c63e39132017346ec6ab6fc427a3c2c34b94dbfc Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 19 Dec 2024 18:27:26 +0100
Subject: [PATCH 021/111] removed some redundant code

---
 tested/nat_translation.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index da3bc60d..1dd449cb 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -29,21 +29,6 @@ def parse_value(
     return value
 
 
-def get_replacement(language: str, translation_stack: list, match: re.Match) -> str:
-    word = match.group(1)
-    current = -1
-    stack = translation_stack[current]
-    while abs(current) <= len(translation_stack) and word not in stack:
-        current -= 1
-        stack = translation_stack[current]
-    if abs(current) <= len(translation_stack):
-        translations = stack[word]
-        assert language in translations
-        word = translations[language]
-
-    return word
-
-
 def flatten_stack(translation_stack: list, language: str) -> dict:
     flattened = {}
     for d in translation_stack:

From a35c49aaa8a6449032c1e25e1189d8b4807996c3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 19 Dec 2024 18:36:26 +0100
Subject: [PATCH 022/111] Adding a few comments

---
 tested/nat_translation.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 1dd449cb..29c0a0cc 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,3 @@
-import re
 import sys
 from typing import cast
 
@@ -19,6 +18,8 @@
 def parse_value(
     value: list | str | int | float | dict, flattened_stack: dict
 ) -> list | str | int | float | dict:
+    # Will format the strings in different values.
+
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
@@ -30,9 +31,12 @@ def parse_value(
 
 
 def flatten_stack(translation_stack: list, language: str) -> dict:
+    # Will transform a list of translation maps into a dict that
+    # has all the keys defined over all the different translation map and will have
+    # the value of the newest definition. In this definition we also chose
+    # the translation of the provided language.
     flattened = {}
     for d in translation_stack:
-
         flattened.update({k: v[language] for k, v in d.items() if language in v})
     return flattened
 

From 10b0eb38d879da983bb6764a7a310b863d185b67 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 14:55:35 +0100
Subject: [PATCH 023/111] Cleaned up code some more and added extra cases for
 input and output tests

---
 tested/nat_translation.py | 128 ++++++++++++++++++++------------------
 1 file changed, 67 insertions(+), 61 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 29c0a0cc..050ec7b6 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -14,10 +14,35 @@
     _validate_testcase_combinations,
 )
 
+def natural_langauge_map_translation(value: YamlObject, language: str):
+    if isinstance(value, NaturalLanguageMap):
+        assert language in value
+        value = value[language]
+    return value
 
-def parse_value(
-    value: list | str | int | float | dict, flattened_stack: dict
-) -> list | str | int | float | dict:
+def translate_input_files(dsl_object: dict, language: str, flattened_stack: dict) -> dict:
+    if (files := dsl_object.get("files")) is not None:
+        # Translation map can happen at the top level.
+        files = natural_langauge_map_translation(files, language)
+        assert isinstance(files, list)
+        for i in range(len(files)):
+            file = files[i]
+
+            # Do the formatting.
+            if isinstance(file, dict):
+                name = file["name"]
+                assert isinstance(name, str)
+                file["name"] = format_string(name, flattened_stack)
+                url = file["url"]
+                assert isinstance(url, str)
+                file["url"] = format_string(url, flattened_stack)
+            files[i] = file
+
+        dsl_object["files"] = files
+    return dsl_object
+
+
+def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
     # Will format the strings in different values.
 
     if isinstance(value, str):
@@ -48,19 +73,15 @@ def format_string(string: str, flattened) -> str:
 def translate_io(
     io_object: YamlObject, key: str, language: str, flat_stack: dict
 ) -> YamlObject:
-    if isinstance(io_object, NaturalLanguageMap):
-        assert language in io_object
-        io_object = io_object[language]
+    # Translate NaturalLanguageMap
+    io_object = natural_langauge_map_translation(io_object, language)
+
     if isinstance(io_object, dict):
-        data = io_object[key]
-        if isinstance(data, dict):
-            assert language in data
-            data = data[language]
+        data = natural_langauge_map_translation(io_object[key], language)
         assert isinstance(data, str)
         io_object[key] = format_string(data, flat_stack)
 
     # Perform translation based of translation stack.
-    print(io_object)
     if isinstance(io_object, str):
         return format_string(io_object, flat_stack)
 
@@ -75,10 +96,8 @@ def translate_testcase(
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
-        # Must use !natural_language
-        if isinstance(expr_stmt, NaturalLanguageMap):
-            assert language in expr_stmt
-            expr_stmt = expr_stmt[language]
+        # Translate NaturalLanguageMap
+        expr_stmt = natural_langauge_map_translation(expr_stmt, language)
 
         # Perform translation based of translation stack.
         if isinstance(expr_stmt, dict):
@@ -90,35 +109,34 @@ def translate_testcase(
 
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
-            if isinstance(stdin_stmt, dict):
-                assert language in stdin_stmt
-                stdin_stmt = stdin_stmt[language]
+            # Translate NaturalLanguageMap
+            stdin_stmt = natural_langauge_map_translation(stdin_stmt, language)
 
             # Perform translation based of translation stack.
             assert isinstance(stdin_stmt, str)
             testcase["stdin"] = format_string(stdin_stmt, flat_stack)
 
+        # Translate NaturalLanguageMap
         arguments = testcase.get("arguments", [])
-        if isinstance(arguments, dict):
-            assert language in arguments
-            arguments = arguments[language]
+        arguments = natural_langauge_map_translation(arguments, language)
 
         # Perform translation based of translation stack.
         assert isinstance(arguments, list)
-        testcase["arguments"] = [
-            format_string(str(arg), flat_stack) for arg in arguments
-        ]
+        testcase["arguments"] = parse_value(arguments, flat_stack)
 
     if (stdout := testcase.get("stdout")) is not None:
-        # Must use !natural_language
         testcase["stdout"] = translate_io(stdout, "data", language, flat_stack)
 
     if (file := testcase.get("file")) is not None:
-        # Must use !natural_language
-        if isinstance(file, NaturalLanguageMap):
-            assert language in file
-            testcase["file"] = file[language]
-        # TODO: SHOULD I ADD SUPPORT FOR TRANSLATION STACK HERE?
+        # Translate NaturalLanguageMap
+        file = natural_langauge_map_translation(file, language)
+
+        assert isinstance(file, dict)
+        file["content"] = format_string(str(file["content"]), flat_stack)
+        file["location"] = format_string(str(file["location"]), flat_stack)
+
+        testcase["file"] = file
+
     if (stderr := testcase.get("stderr")) is not None:
         testcase["stderr"] = translate_io(stderr, "data", language, flat_stack)
 
@@ -128,51 +146,37 @@ def translate_testcase(
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
-            if isinstance(arguments, dict):
-                assert language in arguments
-                arguments = arguments[language]
+            arguments = natural_langauge_map_translation(arguments, language)
 
             # Perform translation based of translation stack.
-            result["arguments"] = [
-                format_string(str(arg), flat_stack) for arg in arguments
-            ]
+            result["arguments"] = parse_value(arguments, flat_stack)
 
             value = result.get("value")
-            # Must use !natural_language
-            if isinstance(value, NaturalLanguageMap):
-                assert language in value
-                value = value[language]
+            value = natural_langauge_map_translation(value, language)
 
-            assert isinstance(value, str)
             result["value"] = parse_value(value, flat_stack)
             testcase["return"] = result
 
         elif isinstance(result, NaturalLanguageMap):
-            # Must use !natural_language
             assert language in result
             testcase["return"] = parse_value(result[language], flat_stack)
         elif result is not None:
             testcase["return"] = parse_value(result, flat_stack)
 
     if (description := testcase.get("description")) is not None:
-        # Must use !natural_language
-        if isinstance(description, NaturalLanguageMap):
-            assert language in description
-            description = description[language]
+        description = natural_langauge_map_translation(description, language)
 
         if isinstance(description, str):
             testcase["description"] = format_string(description, flat_stack)
-
-        if isinstance(description, dict):
+        else:
+            assert isinstance(description, dict)
             dd = description["description"]
-            if isinstance(dd, dict):
-                assert language in dd
-                dd = dd[language]
+            dd = natural_langauge_map_translation(dd, language)
 
             assert isinstance(dd, str)
             description["description"] = format_string(dd, flat_stack)
 
-        testcase["description"] = description
+    testcase = translate_input_files(testcase, language, flat_stack)
 
     return testcase
 
@@ -192,6 +196,8 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
     result = []
     for context in contexts:
         assert isinstance(context, dict)
+
+        # Add translation to stack
         if "translation" in context:
             translation_stack.append(context["translation"])
 
@@ -201,12 +207,12 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
         context[key_to_set] = translate_testcases(
             raw_testcases, language, translation_stack
         )
-        if "files" in context:
-            files = context.get("files")
-            if isinstance(files, NaturalLanguageMap):
-                assert language in files
-                context["files"] = files[language]
+
+        flat_stack = flatten_stack(translation_stack, language)
+        context = translate_input_files(context, language, flat_stack)
         result.append(context)
+
+        # Pop translation from stack
         if "translation" in context:
             translation_stack.pop()
             context.pop("translation")
@@ -217,10 +223,7 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
 def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> YamlDict:
     key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
-
-    if isinstance(name, dict):
-        assert language in name
-        name = name[language]
+    name = natural_langauge_map_translation(name, language)
 
     assert isinstance(name, str)
     tab[key_to_set] = format_string(name, flatten_stack(translation_stack, language))
@@ -283,6 +286,9 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
         if "translation" in dsl_object:
             translation_stack.append(dsl_object["translation"])
             dsl_object.pop("translation")
+
+        flat_stack = flatten_stack(translation_stack, language)
+        dsl_object = translate_input_files(dsl_object, language, flat_stack)
         dsl_object[key_to_set] = translate_tabs(tab_list, language, translation_stack)
         return dsl_object
 

From 3539227486b52423b69ec9e84a9a028bad2f89dd Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 15:33:49 +0100
Subject: [PATCH 024/111] Updated statement/expression case and added
 programmingLanguageMap for consistency

---
 tested/dsl/translate_parser.py | 12 ++++++++++++
 tested/nat_translation.py      | 19 +++++++++----------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 9701a227..9f544943 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -91,6 +91,9 @@ class ReturnOracle(dict):
 class NaturalLanguageMap(dict):
     pass
 
+class ProgrammingLanguageMap(dict):
+    pass
+
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
@@ -104,6 +107,7 @@ class NaturalLanguageMap(dict):
     | ExpressionString
     | ReturnOracle
     | NaturalLanguageMap
+    | ProgrammingLanguageMap
 )
 
 
@@ -158,6 +162,13 @@ def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLangua
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
+def _programming_language_map(loader: yaml.Loader, node: yaml.Node) -> ProgrammingLanguageMap:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A programming language map must be an object, got {result} which is a {type(result)}."
+    return ProgrammingLanguageMap(result)
+
 
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
@@ -170,6 +181,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
     yaml.add_constructor("!expression", _expression_string, loader)
     yaml.add_constructor("!oracle", _return_oracle, loader)
     yaml.add_constructor("!natural_language", _natural_language_map, loader)
+    yaml.add_constructor("!programming_language", _programming_language_map, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 050ec7b6..b54f51b4 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,11 +1,11 @@
 import sys
-from typing import cast
 
 import yaml
 
 from tested.dsl.translate_parser import (
     ExpressionString,
     NaturalLanguageMap,
+    ProgrammingLanguageMap,
     ReturnOracle,
     YamlDict,
     YamlObject,
@@ -96,17 +96,16 @@ def translate_testcase(
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
-        # Translate NaturalLanguageMap
-        expr_stmt = natural_langauge_map_translation(expr_stmt, language)
-
-        # Perform translation based of translation stack.
-        if isinstance(expr_stmt, dict):
-            testcase[key_to_set] = {
-                k: format_string(cast(str, v), flat_stack) for k, v in expr_stmt.items()
+        # Program language translation found
+        if isinstance(expr_stmt, ProgrammingLanguageMap):
+            expr_stmt = {
+                k: natural_langauge_map_translation(v, language) for k, v in expr_stmt.items()
             }
-        elif isinstance(expr_stmt, str):
-            testcase[key_to_set] = format_string(expr_stmt, flat_stack)
+        elif isinstance(expr_stmt, NaturalLanguageMap): # Natural language translation found
+            assert language in expr_stmt
+            expr_stmt = expr_stmt[language]
 
+        testcase[key_to_set] = parse_value(expr_stmt, flat_stack)
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
             # Translate NaturalLanguageMap

From e1180f94dab3abe475c3d59437ed41cf15abe562 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 18:12:09 +0100
Subject: [PATCH 025/111] started added new json schema

---
 tested/dsl/schema-strict-nat-translation.json | 1067 +++++++++++++++++
 tested/nat_translation.py                     |    5 +-
 2 files changed, 1071 insertions(+), 1 deletion(-)
 create mode 100644 tested/dsl/schema-strict-nat-translation.json

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
new file mode 100644
index 00000000..141a42e5
--- /dev/null
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -0,0 +1,1067 @@
+{
+  "$id" : "tested:dsl:schema7",
+  "$schema" : "http://json-schema.org/draft-07/schema#",
+  "title" : "TESTed-DSL",
+  "oneOf" : [
+    {
+      "$ref" : "#/definitions/_rootObject"
+    },
+    {
+      "$ref" : "#/definitions/_tabList"
+    },
+    {
+      "$ref" : "#/definitions/_unitList"
+    }
+  ],
+  "definitions" : {
+    "_rootObject" : {
+      "type" : "object",
+      "oneOf" : [
+        {
+          "required" : [
+            "tabs"
+          ],
+          "not" : {
+            "required" : [
+              "units"
+            ]
+          }
+        },
+        {
+          "required" : [
+            "units"
+          ],
+          "not" : {
+            "required" : [
+              "tabs"
+            ]
+          }
+        }
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "namespace" : {
+          "type" : "string",
+          "description" : "Namespace of the submitted solution, in `snake_case`"
+        },
+        "tabs" : {
+          "$ref" : "#/definitions/_tabList"
+        },
+        "units" : {
+          "$ref" : "#/definitions/_unitList"
+        },
+        "language" : {
+          "description" : "Indicate that all code is in a specific language.",
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/programmingLanguage"
+            },
+            {
+              "const" : "tested"
+            }
+          ]
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the global scope."
+        },
+        "definitions" : {
+          "description" : "Define hashes to use elsewhere.",
+          "type" : "object"
+        },
+        "config": {
+          "$ref": "#/definitions/inheritableConfigObject"
+        }
+      }
+    },
+    "_tabList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/tab"
+      }
+    },
+    "_unitList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/unit"
+      }
+    },
+    "tab" : {
+      "type" : "object",
+      "description" : "A tab in the test suite.",
+      "required" : [
+        "tab"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "hidden" : {
+          "type" : "boolean",
+          "description" : "Defines if the unit/tab is hidden for the student or not"
+        },
+        "tab" : {
+          "oneOf" : [
+            {
+              "type" : "string"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "string"
+              }
+            }
+          ],
+          "description" : "The name of this tab."
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the tab scope."
+        },
+        "definitions" : {
+          "description" : "Define objects to use elsewhere.",
+          "type" : "object"
+        },
+        "config": {
+          "$ref": "#/definitions/inheritableConfigObject"
+        }
+      },
+      "oneOf" : [
+        {
+          "required" : [
+            "contexts"
+          ],
+          "properties" : {
+            "contexts" : {
+              "$ref" : "#/definitions/_contextList"
+            }
+          }
+        },
+        {
+          "required" : [
+            "testcases"
+          ],
+          "properties" : {
+            "testcases" : {
+              "$ref" : "#/definitions/_testcaseList"
+            }
+          }
+        }
+      ]
+    },
+    "unit" : {
+      "type" : "object",
+      "description" : "A unit in the test suite.",
+      "required" : [
+        "unit"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "hidden" : {
+          "type" : "boolean",
+          "description" : "Defines if the unit/tab is hidden for the student or not"
+        },
+        "unit" : {
+          "anyOf" : [
+            {
+              "type" : "string"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "string"
+              }
+            }
+          ],
+          "description" : "The name of this tab."
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the unit scope."
+        },
+        "definitions" : {
+          "description" : "Define objects to use elsewhere.",
+          "type" : "object"
+        },
+        "config": {
+          "$ref": "#/definitions/inheritableConfigObject"
+        }
+      },
+      "oneOf" : [
+        {
+          "required" : [
+            "cases"
+          ],
+          "properties" : {
+            "cases" : {
+              "$ref" : "#/definitions/_caseList"
+            }
+          }
+        },
+        {
+          "required" : [
+            "scripts"
+          ],
+          "properties" : {
+            "scripts" : {
+              "$ref" : "#/definitions/_scriptList"
+            }
+          }
+        }
+      ]
+    },
+    "_contextList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/context"
+      }
+    },
+    "_caseList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/case"
+      }
+    },
+    "_testcaseList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/testcase"
+      }
+    },
+    "_scriptList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/script"
+      }
+    },
+    "context" : {
+      "type" : "object",
+      "description" : "A set of testcase in the same context.",
+      "required" : [
+        "testcases"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the context scope."
+        },
+        "context" : {
+          "type" : "string",
+          "description" : "Description of this context."
+        },
+        "testcases" : {
+          "$ref" : "#/definitions/_testcaseList"
+        }
+      }
+    },
+    "case" : {
+      "type" : "object",
+      "description" : "A test case.",
+      "required" : [
+        "script"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the case scope."
+        },
+        "context" : {
+          "type" : "string",
+          "description" : "Description of this context."
+        },
+        "script" : {
+          "$ref" : "#/definitions/_scriptList"
+        }
+      }
+    },
+    "testcase" : {
+      "type" : "object",
+      "description" : "An individual test for a statement or expression",
+      "additionalProperties" : false,
+      "properties" : {
+        "description" : {
+          "oneOf": [
+            {
+              "$ref" : "#/definitions/message"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/message"
+              }
+            }
+          ]
+        },
+        "stdin" : {
+          "description" : "Stdin for this context",
+          "oneOf": [
+            {
+              "type" : [
+                "string",
+                "number",
+                "integer",
+                "boolean"
+              ]
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            }
+          ]
+        },
+        "arguments" : {
+          "type" : "array",
+          "description" : "Array of program call arguments",
+          "items" : {
+            "type" : [
+              "string",
+              "number",
+              "integer",
+              "boolean"
+            ]
+          }
+        },
+        "statement" : {
+          "description" : "The statement to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "expression" : {
+          "description" : "The expression to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "exception" : {
+          "description" : "Expected exception message",
+          "oneOf" : [
+            {
+              "type" : "string",
+              "description" : "Message of the expected exception."
+            },
+            {
+              "type" : "object",
+              "required" : [
+                "types"
+              ],
+              "properties" : {
+                "message" : {
+                  "type" : "string",
+                  "description" : "Message of the expected exception."
+                },
+                "types" : {
+                  "minProperties" : 1,
+                  "description" : "Language mapping of expected exception types.",
+                  "type" : "object",
+                  "propertyNames" : {
+                    "$ref" : "#/definitions/programmingLanguage"
+                  },
+                  "items" : {
+                    "type" : "string"
+                  }
+                }
+              }
+            }
+          ]
+        },
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "return" : {
+          "description" : "Expected return value",
+          "$ref" : "#/definitions/returnOutputChannel"
+        },
+        "stderr" : {
+          "description" : "Expected output at stderr",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "stdout" : {
+          "description" : "Expected output at stdout",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "file": {
+          "description" : "Expected files generated by the submission.",
+          "$ref" : "#/definitions/fileOutputChannel"
+        },
+        "exit_code" : {
+          "type" : "integer",
+          "description" : "Expected exit code for the run"
+        }
+      }
+    },
+    "script" : {
+      "type" : "object",
+      "description" : "An individual test (script) for a statement or expression",
+      "properties" : {
+        "description" : {
+          "$ref" : "#/definitions/message"
+        },
+        "stdin" : {
+          "description" : "Stdin for this context",
+          "type" : [
+            "string",
+            "number",
+            "integer",
+            "boolean"
+          ]
+        },
+        "arguments" : {
+          "type" : "array",
+          "description" : "Array of program call arguments",
+          "items" : {
+            "type" : [
+              "string",
+              "number",
+              "integer",
+              "boolean"
+            ]
+          }
+        },
+        "statement" : {
+          "description" : "The statement to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "expression" : {
+          "description" : "The expression to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "exception" : {
+          "description" : "Expected exception message",
+          "oneOf" : [
+            {
+              "type" : "string",
+              "description" : "Message of the expected exception."
+            },
+            {
+              "type" : "object",
+              "required" : [
+                "types"
+              ],
+              "properties" : {
+                "message" : {
+                  "type" : "string",
+                  "description" : "Message of the expected exception."
+                },
+                "types" : {
+                  "minProperties" : 1,
+                  "description" : "Language mapping of expected exception types.",
+                  "type" : "object",
+                  "propertyNames" : {
+                    "$ref" : "#/definitions/programmingLanguage"
+                  },
+                  "items" : {
+                    "type" : "string"
+                  }
+                }
+              }
+            }
+          ]
+        },
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "return" : {
+          "description" : "Expected return value",
+          "$ref" : "#/definitions/returnOutputChannel"
+        },
+        "stderr" : {
+          "description" : "Expected output at stderr",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "stdout" : {
+          "description" : "Expected output at stdout",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "exit_code" : {
+          "type" : "integer",
+          "description" : "Expected exit code for the run"
+        }
+      }
+    },
+    "expressionOrStatement" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "format" : "tested-dsl-expression",
+          "description" : "A statement of expression in Python-like syntax as YAML string."
+        },
+        {
+          "description" : "Programming-language-specific statement or expression.",
+          "type" : "object",
+          "minProperties" : 1,
+          "propertyNames" : {
+            "$ref" : "#/definitions/programmingLanguage"
+          },
+          "items" : {
+            "type" : "string",
+            "description" : "A language-specific literal, which will be used verbatim."
+          }
+        }
+      ]
+    },
+    "yamlValueOrPythonExpression" : {
+      "oneOf" : [
+        {
+          "$ref" : "#/definitions/yamlValue"
+        },
+        {
+          "type" : "expression",
+          "format" : "tested-dsl-expression",
+          "description" : "An expression in Python-syntax."
+        }
+      ]
+    },
+    "file" : {
+      "type" : "object",
+      "description" : "A file used in the test suite.",
+      "required" : [
+        "name",
+        "url"
+      ],
+      "properties" : {
+        "name" : {
+          "type" : "string",
+          "description" : "The filename, including the file extension."
+        },
+        "url" : {
+          "type" : "string",
+          "format" : "uri",
+          "description" : "Relative path to the file in the `description` folder of an exercise."
+        }
+      }
+    },
+    "textOutputChannel" : {
+      "anyOf" : [
+        {
+          "$ref" : "#/definitions/textualType"
+        },
+        {
+          "type" : "object",
+          "description" : "Built-in oracle for text values.",
+          "required" : [
+            "data"
+          ],
+          "properties" : {
+            "data" : {
+              "$ref" : "#/definitions/textualType"
+            },
+            "oracle" : {
+              "const" : "builtin"
+            },
+            "config" : {
+              "$ref" : "#/definitions/textConfigurationOptions"
+            }
+          }
+        },
+        {
+          "type" : "object",
+          "description" : "Custom oracle for text values.",
+          "required" : [
+            "oracle",
+            "file",
+            "data"
+          ],
+          "properties" : {
+            "data" : {
+              "$ref" : "#/definitions/textualType"
+            },
+            "oracle" : {
+              "const" : "custom_check"
+            },
+            "file" : {
+              "type" : "string",
+              "description" : "The path to the file containing the custom check function."
+            },
+            "name" : {
+              "type" : "string",
+              "description" : "The name of the custom check function.",
+              "default" : "evaluate"
+            },
+            "arguments" : {
+              "type" : "array",
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+              "items" : {
+                "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
+            }
+          }
+        }
+      ]
+    },
+    "fileOutputChannel": {
+      "anyOf" : [
+        {
+          "type" : "object",
+          "description" : "Built-in oracle for files.",
+          "required" : [
+            "content",
+            "location"
+          ],
+          "properties" : {
+            "content" : {
+              "type" : "string",
+              "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+            },
+            "location" : {
+              "type" : "string",
+              "description" : "Path to where the file generated by the submission should go."
+            },
+            "oracle" : {
+              "const" : "builtin"
+            },
+            "config" : {
+              "$ref" : "#/definitions/fileConfigurationOptions"
+            }
+          }
+        },
+        {
+          "type" : "object",
+          "description" : "Custom oracle for file values.",
+          "required" : [
+            "oracle",
+            "content",
+            "location",
+            "file"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "custom_check"
+            },
+            "content" : {
+              "type" : "string",
+              "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+            },
+            "location" : {
+              "type" : "string",
+              "description" : "Path to where the file generated by the submission should go."
+            },
+            "file" : {
+              "type" : "string",
+              "description" : "The path to the file containing the custom check function."
+            },
+            "name" : {
+              "type" : "string",
+              "description" : "The name of the custom check function.",
+              "default" : "evaluate"
+            },
+            "arguments" : {
+              "type" : "array",
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+              "items" : {
+                "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
+            }
+          }
+        }
+      ]
+    },
+    "returnOutputChannel" : {
+      "oneOf" : [
+        {
+          "$ref" : "#/definitions/yamlValueOrPythonExpression"
+        },
+        {
+          "type" : "oracle",
+          "additionalProperties" : false,
+          "required" : [
+            "value"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "builtin"
+            },
+            "value" : {
+              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            }
+          }
+        },
+        {
+          "type" : "oracle",
+          "additionalProperties" : false,
+          "required" : [
+            "value",
+            "oracle",
+            "file"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "custom_check"
+            },
+            "value" : {
+              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            },
+            "file" : {
+              "type" : "string",
+              "description" : "The path to the file containing the custom check function."
+            },
+            "name" : {
+              "type" : "string",
+              "description" : "The name of the custom check function.",
+              "default" : "evaluate"
+            },
+            "arguments" : {
+              "type" : "array",
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+              "items" : {
+                "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
+            }
+          }
+        },
+        {
+          "type" : "oracle",
+          "additionalProperties" : false,
+          "required" : [
+            "oracle",
+            "functions"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "specific_check"
+            },
+            "functions" : {
+              "minProperties" : 1,
+              "description" : "Language mapping of oracle functions.",
+              "type" : "object",
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "object",
+                "required" : [
+                  "file"
+                ],
+                "properties" : {
+                  "file" : {
+                    "type" : "string",
+                    "description" : "The path to the file containing the custom check function."
+                  },
+                  "name" : {
+                    "type" : "string",
+                    "description" : "The name of the custom check function.",
+                    "default" : "evaluate"
+                  }
+                }
+              }
+            },
+            "arguments" : {
+              "minProperties" : 1,
+              "description" : "Language mapping of oracle arguments.",
+              "type" : "object",
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "array",
+                "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                "items" : {
+                  "type" : "string",
+                  "description" : "A language-specific literal, which will be used verbatim."
+                }
+              }
+            },
+            "value" : {
+              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            }
+          }
+        }
+      ]
+    },
+    "programmingLanguage" : {
+      "type" : "string",
+      "description" : "One of the programming languages supported by TESTed.",
+      "enum" : [
+        "bash",
+        "c",
+        "haskell",
+        "java",
+        "javascript",
+        "typescript",
+        "kotlin",
+        "python",
+        "runhaskell",
+        "csharp"
+      ]
+    },
+    "message" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "description" : "A simple message to display."
+        },
+        {
+          "type" : "object",
+          "required" : [
+            "description"
+          ],
+          "properties" : {
+            "description" : {
+              "oneOf": [
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "type" : "string"
+                  }
+                },
+                {
+                  "type" : "string"
+                }
+              ],
+              "type" : "string",
+              "description" : "The message to display."
+            },
+            "format" : {
+              "type" : "string",
+              "default" : "text",
+              "description" : "The format of the message, either a programming language, 'text' or 'html'."
+            }
+          }
+        }
+      ]
+    },
+    "textConfigurationOptions" : {
+      "type" : "object",
+      "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not",
+      "minProperties" : 1,
+      "properties" : {
+        "applyRounding" : {
+          "description" : "Apply rounding when comparing as float",
+          "type" : "boolean"
+        },
+        "caseInsensitive" : {
+          "description" : "Ignore case when comparing strings",
+          "type" : "boolean"
+        },
+        "ignoreWhitespace" : {
+          "description" : "Ignore trailing whitespace",
+          "type" : "boolean"
+        },
+        "normalizeTrailingNewlines" : {
+          "description" : "Normalize trailing newlines",
+          "type" : "boolean"
+        },
+        "roundTo" : {
+          "description" : "The number of decimals to round at, when applying the rounding on floats",
+          "type" : "integer"
+        },
+        "tryFloatingPoint" : {
+          "description" : "Try comparing text as floating point numbers",
+          "type" : "boolean"
+        },
+        "hideExpected" : {
+          "description" : "Hide the expected value in feedback (default: false), not recommended to use!",
+          "type" : "boolean"
+        }
+      }
+    },
+    "fileConfigurationOptions": {
+      "anyOf" : [
+        {
+          "$ref" : "#/definitions/textConfigurationOptions"
+        },
+        {
+          "type" : "object",
+          "properties" : {
+            "mode": {
+              "type" : "string",
+              "enum" : ["full", "line"],
+              "default" : "full"
+            }
+          }
+        }
+      ]
+    },
+    "textualType" : {
+      "description" : "Simple textual value, converted to string.",
+      "type" : [
+        "string",
+        "number",
+        "integer",
+        "boolean"
+      ]
+    },
+    "yamlValue" : {
+      "description" : "A value represented as YAML.",
+      "not" : {
+        "type" : [
+          "oracle",
+          "expression"
+        ]
+      }
+    },
+    "inheritableConfigObject": {
+      "type": "object",
+      "properties" : {
+        "stdout": {
+          "$ref" : "#/definitions/textConfigurationOptions"
+        },
+        "stderr": {
+          "$ref" : "#/definitions/textConfigurationOptions"
+        },
+        "file": {
+          "$ref" : "#/definitions/fileConfigurationOptions"
+        }
+      }
+    }
+  }
+}
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b54f51b4..b747f363 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -225,7 +225,10 @@ def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> Yaml
     name = natural_langauge_map_translation(name, language)
 
     assert isinstance(name, str)
-    tab[key_to_set] = format_string(name, flatten_stack(translation_stack, language))
+    flat_stack = flatten_stack(translation_stack, language)
+    tab[key_to_set] = format_string(name, flat_stack)
+
+    tab = translate_input_files(tab, language, flat_stack)
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:

From 466ec16a26c11b71093f03d7b982a5a2b1c86983 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 19:49:21 +0100
Subject: [PATCH 026/111] Made some changes to schema

---
 tested/dsl/schema-strict-nat-translation.json | 320 ++++++++++++++----
 tested/dsl/translate_parser.py                |   5 +-
 tested/nat_translation.py                     |  34 +-
 3 files changed, 292 insertions(+), 67 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 141a42e5..339fdff3 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -418,56 +418,57 @@
           ]
         },
         "arguments" : {
-          "type" : "array",
-          "description" : "Array of program call arguments",
-          "items" : {
-            "type" : [
-              "string",
-              "number",
-              "integer",
-              "boolean"
-            ]
-          }
+          "oneOf": [
+            {
+              "type" : "array",
+              "items" : {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "type" : [
+                    "string",
+                    "number",
+                    "integer",
+                    "boolean"
+                  ]
+                }
+              }
+            }
+          ],
+          "description" : "Array of program call arguments"
         },
         "statement" : {
           "description" : "The statement to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "expression" : {
           "description" : "The expression to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "exception" : {
           "description" : "Expected exception message",
           "oneOf" : [
             {
-              "type" : "string",
-              "description" : "Message of the expected exception."
+              "$ref" : "#/definitions/exceptionChannel"
             },
             {
-              "type" : "object",
-              "required" : [
-                "types"
-              ],
-              "properties" : {
-                "message" : {
-                  "type" : "string",
-                  "description" : "Message of the expected exception."
-                },
-                "types" : {
-                  "minProperties" : 1,
-                  "description" : "Language mapping of expected exception types.",
-                  "type" : "object",
-                  "propertyNames" : {
-                    "$ref" : "#/definitions/programmingLanguage"
-                  },
-                  "items" : {
-                    "type" : "string"
-                  }
-                }
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ]
+          ],
+          "$ref" : "#/definitions/exceptionChannel"
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
@@ -491,19 +492,60 @@
         },
         "return" : {
           "description" : "Expected return value",
-          "$ref" : "#/definitions/returnOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/returnOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/returnOutputChannel"
+              }
+            }
+          ]
+
         },
         "stderr" : {
           "description" : "Expected output at stderr",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
         },
         "stdout" : {
           "description" : "Expected output at stdout",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
         },
         "file": {
           "description" : "Expected files generated by the submission.",
-          "$ref" : "#/definitions/fileOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/fileOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/fileOutputChannel"
+              }
+            }
+          ]
         },
         "exit_code" : {
           "type" : "integer",
@@ -638,6 +680,33 @@
         }
       ]
     },
+    "expressionOrStatementWithNatTranslation" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "format" : "tested-dsl-expression",
+          "description" : "A statement of expression in Python-like syntax as YAML string."
+        },
+        {
+          "description" : "Programming-language-specific statement or expression.",
+          "type" : "object",
+          "minProperties" : 1,
+          "propertyNames" : {
+            "$ref" : "#/definitions/programmingLanguage"
+          },
+          "items" : {
+            "type" : "string",
+            "description" : "A language-specific literal, which will be used verbatim."
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "$ref" : "#/definitions/expressionOrStatement"
+          }
+        }
+      ]
+    },
     "yamlValueOrPythonExpression" : {
       "oneOf" : [
         {
@@ -669,6 +738,47 @@
         }
       }
     },
+    "exceptionChannel" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "description" : "Message of the expected exception."
+        },
+        {
+          "type" : "object",
+          "required" : [
+            "types"
+          ],
+          "properties" : {
+            "message" : {
+              "oneOf" : [
+                {
+                  "type" : "string"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "type" : "string"
+                  }
+                }
+              ],
+              "description" : "Message of the expected exception."
+            },
+            "types" : {
+              "minProperties" : 1,
+              "description" : "Language mapping of expected exception types.",
+              "type" : "object",
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "string"
+              }
+            }
+          }
+        }
+      ]
+    },
     "textOutputChannel" : {
       "anyOf" : [
         {
@@ -682,7 +792,17 @@
           ],
           "properties" : {
             "data" : {
-              "$ref" : "#/definitions/textualType"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/textualType"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textualType"
+                  }
+                }
+              ]
             },
             "oracle" : {
               "const" : "builtin"
@@ -702,7 +822,17 @@
           ],
           "properties" : {
             "data" : {
-              "$ref" : "#/definitions/textualType"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/textualType"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textualType"
+                  }
+                }
+              ]
             },
             "oracle" : {
               "const" : "custom_check"
@@ -824,7 +954,17 @@
               "const" : "builtin"
             },
             "value" : {
-              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                }
+              ]
             }
           }
         },
@@ -841,7 +981,17 @@
               "const" : "custom_check"
             },
             "value" : {
-              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                }
+              ]
             },
             "file" : {
               "type" : "string",
@@ -853,11 +1003,24 @@
               "default" : "evaluate"
             },
             "arguments" : {
-              "type" : "array",
-              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
-              "items" : {
-                "$ref" : "#/definitions/yamlValueOrPythonExpression"
-              }
+              "oneOf" : [
+                {
+                  "type" : "array",
+                  "items" : {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                    }
+                  }
+                }
+              ],
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
             },
             "languages": {
               "type" : "array",
@@ -905,23 +1068,56 @@
               }
             },
             "arguments" : {
-              "minProperties" : 1,
-              "description" : "Language mapping of oracle arguments.",
-              "type" : "object",
-              "propertyNames" : {
-                "$ref" : "#/definitions/programmingLanguage"
-              },
-              "items" : {
-                "type" : "array",
-                "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
-                "items" : {
-                  "type" : "string",
-                  "description" : "A language-specific literal, which will be used verbatim."
+              "oneOf" : [
+                {
+                  "minProperties" : 1,
+                  "description" : "Language mapping of oracle arguments.",
+                  "type" : "object",
+                  "propertyNames" : {
+                    "$ref" : "#/definitions/programmingLanguage"
+                  },
+                  "items" : {
+                    "type" : "array",
+                    "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                    "items" : {
+                      "type" : "string",
+                      "description" : "A language-specific literal, which will be used verbatim."
+                    }
+                  }
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "minProperties" : 1,
+                    "description" : "Language mapping of oracle arguments.",
+                    "type" : "object",
+                    "propertyNames" : {
+                      "$ref" : "#/definitions/programmingLanguage"
+                    },
+                    "items" : {
+                      "type" : "array",
+                      "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                      "items" : {
+                        "type" : "string",
+                        "description" : "A language-specific literal, which will be used verbatim."
+                      }
+                    }
+                  }
                 }
-              }
+              ]
             },
             "value" : {
-              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                }
+              ]
             }
           }
         }
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 9f544943..97a88be8 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -220,6 +220,9 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool:
 def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
+def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, NaturalLanguageMap)
+
 
 def test(value: object) -> bool:
     if not isinstance(value, str):
@@ -241,7 +244,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
     original_validator: Type[Validator] = validator_for(schema_object)
     type_checker = original_validator.TYPE_CHECKER.redefine(
         "oracle", is_oracle
-    ).redefine("expression", is_expression)
+    ).redefine("expression", is_expression).redefine("natural_language", is_natural_language_map)
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b747f363..aea2b8b9 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -11,9 +11,35 @@
     YamlObject,
     _parse_yaml,
     _validate_dsl,
-    _validate_testcase_combinations,
+    _validate_testcase_combinations, load_schema_validator,
+    convert_validation_error_to_group, DslValidationError,
 )
 
+
+def validate_pre_dsl(dsl_object: YamlObject):
+    """
+    Validate a DSl object.
+
+    :param dsl_object: The object to validate.
+    :return: True if valid, False otherwise.
+    """
+    _SCHEMA_VALIDATOR = load_schema_validator("schema-strict-nat-translation.json")
+    errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
+    if len(errors) == 1:
+        message = (
+            "Validating the DSL resulted in an error. "
+            "The most specific sub-exception is often the culprit. "
+        )
+        error = convert_validation_error_to_group(errors[0])
+        if isinstance(error, ExceptionGroup):
+            raise ExceptionGroup(message, error.exceptions)
+        else:
+            raise DslValidationError(message + str(error)) from error
+    elif len(errors) > 1:
+        the_errors = [convert_validation_error_to_group(e) for e in errors]
+        message = "Validating the DSL resulted in some errors."
+        raise ExceptionGroup(message, the_errors)
+
 def natural_langauge_map_translation(value: YamlObject, language: str):
     if isinstance(value, NaturalLanguageMap):
         assert language in value
@@ -78,11 +104,10 @@ def translate_io(
 
     if isinstance(io_object, dict):
         data = natural_langauge_map_translation(io_object[key], language)
-        assert isinstance(data, str)
-        io_object[key] = format_string(data, flat_stack)
+        io_object[key] = parse_value(data, flat_stack)
 
     # Perform translation based of translation stack.
-    if isinstance(io_object, str):
+    elif isinstance(io_object, str):
         return format_string(io_object, flat_stack)
 
     return io_object
@@ -322,6 +347,7 @@ def expression_representer(dumper, data):
     path = sys.argv[1]
     lang = sys.argv[2]
     new_yaml = parse_yaml(path)
+    validate_pre_dsl(new_yaml)
     translated_dsl = translate_dsl(new_yaml, lang)
     yaml_string = convert_to_yaml(translated_dsl)
     print(yaml_string)

From 49079a0cc5fb5adf26f728d0f97f3b0bbba5affc Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 13:01:08 +0100
Subject: [PATCH 027/111] fixed some bugs in the schema

---
 tested/dsl/schema-strict-nat-translation.json | 165 +++++++++++++-----
 1 file changed, 117 insertions(+), 48 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 339fdff3..02c77148 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -558,68 +558,94 @@
       "description" : "An individual test (script) for a statement or expression",
       "properties" : {
         "description" : {
-          "$ref" : "#/definitions/message"
+          "oneOf": [
+            {
+              "$ref" : "#/definitions/message"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/message"
+              }
+            }
+          ]
         },
         "stdin" : {
           "description" : "Stdin for this context",
-          "type" : [
-            "string",
-            "number",
-            "integer",
-            "boolean"
+          "oneOf": [
+            {
+              "type" : [
+                "string",
+                "number",
+                "integer",
+                "boolean"
+              ]
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            }
           ]
         },
         "arguments" : {
-          "type" : "array",
-          "description" : "Array of program call arguments",
-          "items" : {
-            "type" : [
-              "string",
-              "number",
-              "integer",
-              "boolean"
-            ]
-          }
+          "oneOf": [
+            {
+              "type" : "array",
+              "items" : {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "type" : [
+                    "string",
+                    "number",
+                    "integer",
+                    "boolean"
+                  ]
+                }
+              }
+            }
+          ],
+          "description" : "Array of program call arguments"
         },
         "statement" : {
           "description" : "The statement to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "expression" : {
           "description" : "The expression to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "exception" : {
           "description" : "Expected exception message",
           "oneOf" : [
             {
-              "type" : "string",
-              "description" : "Message of the expected exception."
+              "$ref" : "#/definitions/exceptionChannel"
             },
             {
-              "type" : "object",
-              "required" : [
-                "types"
-              ],
-              "properties" : {
-                "message" : {
-                  "type" : "string",
-                  "description" : "Message of the expected exception."
-                },
-                "types" : {
-                  "minProperties" : 1,
-                  "description" : "Language mapping of expected exception types.",
-                  "type" : "object",
-                  "propertyNames" : {
-                    "$ref" : "#/definitions/programmingLanguage"
-                  },
-                  "items" : {
-                    "type" : "string"
-                  }
-                }
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ]
+          ],
+          "$ref" : "#/definitions/exceptionChannel"
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
@@ -643,15 +669,60 @@
         },
         "return" : {
           "description" : "Expected return value",
-          "$ref" : "#/definitions/returnOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/returnOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/returnOutputChannel"
+              }
+            }
+          ]
+
         },
         "stderr" : {
           "description" : "Expected output at stderr",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
         },
         "stdout" : {
           "description" : "Expected output at stdout",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
+        },
+        "file": {
+          "description" : "Expected files generated by the submission.",
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/fileOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/fileOutputChannel"
+              }
+            }
+          ]
         },
         "exit_code" : {
           "type" : "integer",
@@ -663,7 +734,6 @@
       "oneOf" : [
         {
           "type" : "string",
-          "format" : "tested-dsl-expression",
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
@@ -684,7 +754,6 @@
       "oneOf" : [
         {
           "type" : "string",
-          "format" : "tested-dsl-expression",
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
@@ -1163,7 +1232,6 @@
                   "type" : "string"
                 }
               ],
-              "type" : "string",
               "description" : "The message to display."
             },
             "format" : {
@@ -1241,7 +1309,8 @@
       "not" : {
         "type" : [
           "oracle",
-          "expression"
+          "expression",
+          "natural_language"
         ]
       }
     },

From 4174beb68ee95f304d98cd8a1b1eef094e93efef Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 13:42:20 +0100
Subject: [PATCH 028/111] fixed some bugs and fixed the tests

---
 tested/dsl/schema-strict-nat-translation.json |  6 ++--
 tested/dsl/translate_parser.py                | 15 +++++++---
 tested/nat_translation.py                     | 19 +++++++++----
 tests/test_dsl_yaml.py                        | 28 ++++++++++---------
 4 files changed, 42 insertions(+), 26 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 02c77148..68901530 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -467,8 +467,7 @@
                 "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ],
-          "$ref" : "#/definitions/exceptionChannel"
+          ]
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
@@ -644,8 +643,7 @@
                 "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ],
-          "$ref" : "#/definitions/exceptionChannel"
+          ]
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 97a88be8..b1452521 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -91,6 +91,7 @@ class ReturnOracle(dict):
 class NaturalLanguageMap(dict):
     pass
 
+
 class ProgrammingLanguageMap(dict):
     pass
 
@@ -162,7 +163,10 @@ def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLangua
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
-def _programming_language_map(loader: yaml.Loader, node: yaml.Node) -> ProgrammingLanguageMap:
+
+def _programming_language_map(
+    loader: yaml.Loader, node: yaml.Node
+) -> ProgrammingLanguageMap:
     result = _parse_yaml_value(loader, node)
     assert isinstance(
         result, dict
@@ -220,6 +224,7 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool:
 def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
+
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
@@ -242,9 +247,11 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         schema_object = json.load(schema_file)
 
     original_validator: Type[Validator] = validator_for(schema_object)
-    type_checker = original_validator.TYPE_CHECKER.redefine(
-        "oracle", is_oracle
-    ).redefine("expression", is_expression).redefine("natural_language", is_natural_language_map)
+    type_checker = (
+        original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
+        .redefine("expression", is_expression)
+        .redefine("natural_language", is_natural_language_map)
+    )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index aea2b8b9..c0dd81ce 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -3,6 +3,7 @@
 import yaml
 
 from tested.dsl.translate_parser import (
+    DslValidationError,
     ExpressionString,
     NaturalLanguageMap,
     ProgrammingLanguageMap,
@@ -11,8 +12,9 @@
     YamlObject,
     _parse_yaml,
     _validate_dsl,
-    _validate_testcase_combinations, load_schema_validator,
-    convert_validation_error_to_group, DslValidationError,
+    _validate_testcase_combinations,
+    convert_validation_error_to_group,
+    load_schema_validator,
 )
 
 
@@ -40,13 +42,17 @@ def validate_pre_dsl(dsl_object: YamlObject):
         message = "Validating the DSL resulted in some errors."
         raise ExceptionGroup(message, the_errors)
 
+
 def natural_langauge_map_translation(value: YamlObject, language: str):
     if isinstance(value, NaturalLanguageMap):
         assert language in value
         value = value[language]
     return value
 
-def translate_input_files(dsl_object: dict, language: str, flattened_stack: dict) -> dict:
+
+def translate_input_files(
+    dsl_object: dict, language: str, flattened_stack: dict
+) -> dict:
     if (files := dsl_object.get("files")) is not None:
         # Translation map can happen at the top level.
         files = natural_langauge_map_translation(files, language)
@@ -124,9 +130,12 @@ def translate_testcase(
         # Program language translation found
         if isinstance(expr_stmt, ProgrammingLanguageMap):
             expr_stmt = {
-                k: natural_langauge_map_translation(v, language) for k, v in expr_stmt.items()
+                k: natural_langauge_map_translation(v, language)
+                for k, v in expr_stmt.items()
             }
-        elif isinstance(expr_stmt, NaturalLanguageMap): # Natural language translation found
+        elif isinstance(
+            expr_stmt, NaturalLanguageMap
+        ):  # Natural language translation found
             assert language in expr_stmt
             expr_stmt = expr_stmt[language]
 
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 87900fae..f4bd9a8d 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1367,7 +1367,7 @@ def test_natural_translate_unit_test():
             oracle: "custom_check"
             file: "test.py"
             name: "evaluate_test"
-            arguments:
+            arguments: !natural_language
               en: ["The value", "is OK!", "is not OK!"]
               nl: ["Het {result}", "is OK!", "is niet OK!"]
           description: !natural_language
@@ -1391,7 +1391,7 @@ def test_natural_translate_unit_test():
           - expression: 'result'
             return: '11_{elf}'
             description:
-              description:
+              description: !natural_language
                 en: "Eleven_{elf}"
                 nl: "Elf_{elf}"
               format: "code"
@@ -1401,11 +1401,13 @@ def test_natural_translate_unit_test():
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
-      - expression:
+      - expression: !programming_language
           javascript: "{animal}_javascript(1 + 1)"
           typescript: "{animal}_typescript(1 + 1)"
           java: "Submission.{animal}_java(1 + 1)"
-          python: "{animal}_python(1 + 1)"
+          python: !natural_language
+            en: "{animal}_python_en(1 + 1)"
+            nl: "{animal}_python_nl(1 + 1)"
         return: 2
 """.strip()
     translated_yaml_str = """
@@ -1447,7 +1449,7 @@ def test_natural_translate_unit_test():
       javascript: animals_javascript(1 + 1)
       typescript: animals_typescript(1 + 1)
       java: Submission.animals_java(1 + 1)
-      python: animals_python(1 + 1)
+      python: animals_python_en(1 + 1)
     return: 2
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
@@ -1462,7 +1464,7 @@ def test_natural_translate_io_test():
     # Everywhere else it isn't.
     yaml_str = """
 units:
-  - unit:
+  - unit: !natural_language
       en: "Arguments"
       nl: "Argumenten"
     translation:
@@ -1471,10 +1473,10 @@ def test_natural_translate_io_test():
         nl: "gebruiker"
     cases:
       - script:
-        - stdin:
+        - stdin: !natural_language
             en: "User_{User}"
             nl: "Gebruiker_{User}"
-          arguments:
+          arguments: !natural_language
             en: [ "input_{User}", "output_{User}" ]
             nl: [ "invoer_{User}", "uitvoer_{User}" ]
           stdout: !natural_language
@@ -1486,26 +1488,26 @@ def test_natural_translate_io_test():
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
-        - stdin:
+        - stdin: !natural_language
             en: "Friend of {User}"
             nl: "Vriend van {User}"
-          arguments:
+          arguments: !natural_language
             en: [ "input", "output" ]
             nl: [ "invoer", "uitvoer" ]
           stdout:
-            data:
+            data: !natural_language
               en: "Hi Friend of {User}"
               nl: "Hallo Vriend van {User}"
             config:
               ignoreWhitespace: true
           stderr:
-            data:
+            data: !natural_language
               en: "Nothing to see here {User}"
               nl: "Hier is niets te zien {User}"
             config:
               ignoreWhitespace: true
           exception:
-            message:
+            message: !natural_language
               en: "Does not look good {User}"
               nl: "Ziet er niet goed uit {User}"
             types:

From 1a79a82912db7a7dc1f1f30f524620ebc5f95a88 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 15:56:33 +0100
Subject: [PATCH 029/111] fixed an edge case and made an extra test for it.

---
 tested/dsl/schema-strict-nat-translation.json |  7 ++---
 tested/dsl/translate_parser.py                |  4 +++
 tested/nat_translation.py                     |  2 +-
 tests/test_dsl_yaml.py                        | 27 ++++++++++++++++++-
 4 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 68901530..59cbc1c2 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -736,7 +736,7 @@
         },
         {
           "description" : "Programming-language-specific statement or expression.",
-          "type" : "object",
+          "type" : "programming_language",
           "minProperties" : 1,
           "propertyNames" : {
             "$ref" : "#/definitions/programmingLanguage"
@@ -756,7 +756,7 @@
         },
         {
           "description" : "Programming-language-specific statement or expression.",
-          "type" : "object",
+          "type" : "programming_language",
           "minProperties" : 1,
           "propertyNames" : {
             "$ref" : "#/definitions/programmingLanguage"
@@ -1308,7 +1308,8 @@
         "type" : [
           "oracle",
           "expression",
-          "natural_language"
+          "natural_language",
+          "programming_language"
         ]
       }
     },
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index b1452521..643b1dac 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -228,6 +228,9 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
+def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, ProgrammingLanguageMap)
+
 
 def test(value: object) -> bool:
     if not isinstance(value, str):
@@ -251,6 +254,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
         .redefine("expression", is_expression)
         .redefine("natural_language", is_natural_language_map)
+        .redefine("programming_language", is_programming_language_map)
     )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index c0dd81ce..52dc7ca0 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -75,8 +75,8 @@ def translate_input_files(
 
 
 def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
-    # Will format the strings in different values.
 
+    # Will format the strings in different values.
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index f4bd9a8d..c4638faa 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,8 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl
+from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl, \
+    validate_pre_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1573,3 +1574,27 @@ def test_translate_parse():
     }
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value
+
+def test_wrong_natural_translation_suite():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python:
+        en: animals_python_en(1 + 1)
+        nl: animals_python_nl(1 + 1)
+    return: 2
+    """.strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    try:
+        validate_pre_dsl(parsed_yaml)
+    except ExceptionGroup:
+        print("As expected")
+    else:
+        assert False, "Expected ExceptionGroup, but no exception was raised"

From 81ce1614cffcc2b04fe2a235ab9858318a0f00e9 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 16:15:57 +0100
Subject: [PATCH 030/111] added the actual writing to a file.

---
 tested/dsl/translate_parser.py |  1 +
 tested/nat_translation.py      | 30 +++++++++++++++++++++---------
 tests/test_dsl_yaml.py         |  9 +++++++--
 3 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 643b1dac..1c0d1e5d 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -228,6 +228,7 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
+
 def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ProgrammingLanguageMap)
 
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 52dc7ca0..89c52f32 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,5 @@
 import sys
+from pathlib import Path
 
 import yaml
 
@@ -329,13 +330,21 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
         return dsl_object
 
 
-def parse_yaml(yaml_path: str) -> YamlObject:
+def parse_yaml(yaml_path: Path) -> YamlObject:
     with open(yaml_path, "r") as stream:
         result = _parse_yaml(stream.read())
 
     return result
 
 
+def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
+    file_name = yaml_path.name
+    split_name = file_name.split(".")
+    path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
+    with open(path_to_new_yaml, "w") as yaml_file:
+        yaml_file.write(yaml_string)
+
+
 def convert_to_yaml(yaml_object: YamlObject) -> str:
     def oracle_representer(dumper, data):
         return dumper.represent_mapping("!oracle", data)
@@ -349,15 +358,18 @@ def expression_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-if __name__ == "__main__":
-    n = len(sys.argv)
-    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
-
-    path = sys.argv[1]
-    lang = sys.argv[2]
+def run(path: Path, language: str):
     new_yaml = parse_yaml(path)
     validate_pre_dsl(new_yaml)
-    translated_dsl = translate_dsl(new_yaml, lang)
+    translated_dsl = translate_dsl(new_yaml, language)
     yaml_string = convert_to_yaml(translated_dsl)
-    print(yaml_string)
     _validate_dsl(_parse_yaml(yaml_string))
+
+    generate_new_yaml(path, yaml_string, language)
+
+
+if __name__ == "__main__":
+    n = len(sys.argv)
+    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+
+    run(Path(sys.argv[1]), sys.argv[2])
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index c4638faa..14034b8a 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,8 +21,12 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl, \
-    validate_pre_dsl
+from tested.nat_translation import (
+    convert_to_yaml,
+    parse_value,
+    translate_dsl,
+    validate_pre_dsl,
+)
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1575,6 +1579,7 @@ def test_translate_parse():
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value
 
+
 def test_wrong_natural_translation_suite():
     yaml_str = """
 tabs:

From 64a00cd93258d4f10d3b6fd183cf02547e634965 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 4 Jan 2025 16:45:06 +0100
Subject: [PATCH 031/111] changed formatter to jinja

---
 tested/nat_translation.py | 175 +++++++++++++++++++++++---------------
 tests/test_dsl_yaml.py    | 101 ++++++++++++----------
 2 files changed, 163 insertions(+), 113 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 89c52f32..f555976b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,6 +2,7 @@
 from pathlib import Path
 
 import yaml
+from jinja2 import Environment, Template
 
 from tested.dsl.translate_parser import (
     DslValidationError,
@@ -44,7 +45,7 @@ def validate_pre_dsl(dsl_object: YamlObject):
         raise ExceptionGroup(message, the_errors)
 
 
-def natural_langauge_map_translation(value: YamlObject, language: str):
+def natural_language_map_translation(value: YamlObject, language: str):
     if isinstance(value, NaturalLanguageMap):
         assert language in value
         value = value[language]
@@ -52,11 +53,11 @@ def natural_langauge_map_translation(value: YamlObject, language: str):
 
 
 def translate_input_files(
-    dsl_object: dict, language: str, flattened_stack: dict
+    dsl_object: dict, language: str, flattened_stack: dict, env: Environment
 ) -> dict:
     if (files := dsl_object.get("files")) is not None:
         # Translation map can happen at the top level.
-        files = natural_langauge_map_translation(files, language)
+        files = natural_language_map_translation(files, language)
         assert isinstance(files, list)
         for i in range(len(files)):
             file = files[i]
@@ -65,25 +66,27 @@ def translate_input_files(
             if isinstance(file, dict):
                 name = file["name"]
                 assert isinstance(name, str)
-                file["name"] = format_string(name, flattened_stack)
+                file["name"] = format_string(name, flattened_stack, env)
                 url = file["url"]
                 assert isinstance(url, str)
-                file["url"] = format_string(url, flattened_stack)
+                file["url"] = format_string(url, flattened_stack, env)
             files[i] = file
 
         dsl_object["files"] = files
     return dsl_object
 
 
-def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
+def parse_value(
+    value: YamlObject, flattened_stack: dict, env: Environment
+) -> YamlObject:
 
     # Will format the strings in different values.
     if isinstance(value, str):
-        return format_string(value, flattened_stack)
+        return format_string(value, flattened_stack, env)
     elif isinstance(value, dict):
-        return {k: parse_value(v, flattened_stack) for k, v in value.items()}
+        return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
     elif isinstance(value, list):
-        return [parse_value(v, flattened_stack) for v in value]
+        return [parse_value(v, flattened_stack, env) for v in value]
 
     return value
 
@@ -99,29 +102,34 @@ def flatten_stack(translation_stack: list, language: str) -> dict:
     return flattened
 
 
-def format_string(string: str, flattened) -> str:
-    return string.format(**flattened)
+def format_string(string: str, translations: dict, env: Environment) -> str:
+    template = env.from_string(string)
+    result = template.render(translations)
+    # print(f"jinja result: {result}")
+
+    # return string.format(**translations)
+    return result
 
 
 def translate_io(
-    io_object: YamlObject, key: str, language: str, flat_stack: dict
+    io_object: YamlObject, key: str, language: str, flat_stack: dict, env: Environment
 ) -> YamlObject:
     # Translate NaturalLanguageMap
-    io_object = natural_langauge_map_translation(io_object, language)
+    io_object = natural_language_map_translation(io_object, language)
 
     if isinstance(io_object, dict):
-        data = natural_langauge_map_translation(io_object[key], language)
-        io_object[key] = parse_value(data, flat_stack)
+        data = natural_language_map_translation(io_object[key], language)
+        io_object[key] = parse_value(data, flat_stack, env)
 
     # Perform translation based of translation stack.
     elif isinstance(io_object, str):
-        return format_string(io_object, flat_stack)
+        return format_string(io_object, flat_stack, env)
 
     return io_object
 
 
 def translate_testcase(
-    testcase: YamlDict, language: str, translation_stack: list
+    testcase: YamlDict, language: str, translation_stack: list, env: Environment
 ) -> YamlDict:
     _validate_testcase_combinations(testcase)
     flat_stack = flatten_stack(translation_stack, language)
@@ -131,7 +139,7 @@ def translate_testcase(
         # Program language translation found
         if isinstance(expr_stmt, ProgrammingLanguageMap):
             expr_stmt = {
-                k: natural_langauge_map_translation(v, language)
+                k: natural_language_map_translation(v, language)
                 for k, v in expr_stmt.items()
             }
         elif isinstance(
@@ -140,159 +148,169 @@ def translate_testcase(
             assert language in expr_stmt
             expr_stmt = expr_stmt[language]
 
-        testcase[key_to_set] = parse_value(expr_stmt, flat_stack)
+        testcase[key_to_set] = parse_value(expr_stmt, flat_stack, env)
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
             # Translate NaturalLanguageMap
-            stdin_stmt = natural_langauge_map_translation(stdin_stmt, language)
+            stdin_stmt = natural_language_map_translation(stdin_stmt, language)
 
             # Perform translation based of translation stack.
             assert isinstance(stdin_stmt, str)
-            testcase["stdin"] = format_string(stdin_stmt, flat_stack)
+            testcase["stdin"] = format_string(stdin_stmt, flat_stack, env)
 
         # Translate NaturalLanguageMap
         arguments = testcase.get("arguments", [])
-        arguments = natural_langauge_map_translation(arguments, language)
+        arguments = natural_language_map_translation(arguments, language)
 
         # Perform translation based of translation stack.
         assert isinstance(arguments, list)
-        testcase["arguments"] = parse_value(arguments, flat_stack)
+        testcase["arguments"] = parse_value(arguments, flat_stack, env)
 
     if (stdout := testcase.get("stdout")) is not None:
-        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack)
+        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack, env)
 
     if (file := testcase.get("file")) is not None:
         # Translate NaturalLanguageMap
-        file = natural_langauge_map_translation(file, language)
+        file = natural_language_map_translation(file, language)
 
         assert isinstance(file, dict)
-        file["content"] = format_string(str(file["content"]), flat_stack)
-        file["location"] = format_string(str(file["location"]), flat_stack)
+        file["content"] = format_string(str(file["content"]), flat_stack, env)
+        file["location"] = format_string(str(file["location"]), flat_stack, env)
 
         testcase["file"] = file
 
     if (stderr := testcase.get("stderr")) is not None:
-        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack)
+        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack, env)
 
     if (exception := testcase.get("exception")) is not None:
-        testcase["exception"] = translate_io(exception, "message", language, flat_stack)
+        testcase["exception"] = translate_io(
+            exception, "message", language, flat_stack, env
+        )
 
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
-            arguments = natural_langauge_map_translation(arguments, language)
+            arguments = natural_language_map_translation(arguments, language)
 
             # Perform translation based of translation stack.
-            result["arguments"] = parse_value(arguments, flat_stack)
+            result["arguments"] = parse_value(arguments, flat_stack, env)
 
             value = result.get("value")
-            value = natural_langauge_map_translation(value, language)
+            value = natural_language_map_translation(value, language)
 
-            result["value"] = parse_value(value, flat_stack)
+            result["value"] = parse_value(value, flat_stack, env)
             testcase["return"] = result
 
         elif isinstance(result, NaturalLanguageMap):
             assert language in result
-            testcase["return"] = parse_value(result[language], flat_stack)
+            testcase["return"] = parse_value(result[language], flat_stack, env)
         elif result is not None:
-            testcase["return"] = parse_value(result, flat_stack)
+            testcase["return"] = parse_value(result, flat_stack, env)
 
     if (description := testcase.get("description")) is not None:
-        description = natural_langauge_map_translation(description, language)
+        description = natural_language_map_translation(description, language)
 
         if isinstance(description, str):
-            testcase["description"] = format_string(description, flat_stack)
+            testcase["description"] = format_string(description, flat_stack, env)
         else:
             assert isinstance(description, dict)
             dd = description["description"]
-            dd = natural_langauge_map_translation(dd, language)
+            dd = natural_language_map_translation(dd, language)
 
             assert isinstance(dd, str)
-            description["description"] = format_string(dd, flat_stack)
+            description["description"] = format_string(dd, flat_stack, env)
 
-    testcase = translate_input_files(testcase, language, flat_stack)
+    testcase = translate_input_files(testcase, language, flat_stack, env)
 
     return testcase
 
 
 def translate_testcases(
-    testcases: list, language: str, translation_stack: list
+    testcases: list, language: str, translation_stack: list, env: Environment
 ) -> list:
     result = []
     for testcase in testcases:
         assert isinstance(testcase, dict)
-        result.append(translate_testcase(testcase, language, translation_stack))
+        result.append(translate_testcase(testcase, language, translation_stack, env))
 
     return result
 
 
-def translate_contexts(contexts: list, language: str, translation_stack: list) -> list:
+def translate_contexts(
+    contexts: list, language: str, translation_stack: list, env: Environment
+) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
 
         # Add translation to stack
-        if "translation" in context:
-            translation_stack.append(context["translation"])
+        if "translations" in context:
+            translation_stack.append(context["translations"])
 
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
         context[key_to_set] = translate_testcases(
-            raw_testcases, language, translation_stack
+            raw_testcases, language, translation_stack, env
         )
 
         flat_stack = flatten_stack(translation_stack, language)
-        context = translate_input_files(context, language, flat_stack)
+        context = translate_input_files(context, language, flat_stack, env)
         result.append(context)
 
         # Pop translation from stack
-        if "translation" in context:
+        if "translations" in context:
             translation_stack.pop()
-            context.pop("translation")
+            context.pop("translations")
 
     return result
 
 
-def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> YamlDict:
+def translate_tab(
+    tab: YamlDict, language: str, translation_stack: list, env: Environment
+) -> YamlDict:
     key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
-    name = natural_langauge_map_translation(name, language)
+    name = natural_language_map_translation(name, language)
 
     assert isinstance(name, str)
     flat_stack = flatten_stack(translation_stack, language)
-    tab[key_to_set] = format_string(name, flat_stack)
+    tab[key_to_set] = format_string(name, flat_stack, env)
 
-    tab = translate_input_files(tab, language, flat_stack)
+    tab = translate_input_files(tab, language, flat_stack, env)
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:
         assert isinstance(tab["contexts"], list)
         tab["contexts"] = translate_contexts(
-            tab["contexts"], language, translation_stack
+            tab["contexts"], language, translation_stack, env
         )
     elif "cases" in tab:
         assert "unit" in tab
         # We have testcases N.S. / contexts O.S.
         assert isinstance(tab["cases"], list)
-        tab["cases"] = translate_contexts(tab["cases"], language, translation_stack)
+        tab["cases"] = translate_contexts(
+            tab["cases"], language, translation_stack, env
+        )
     elif "testcases" in tab:
         # We have scripts N.S. / testcases O.S.
         assert "tab" in tab
         assert isinstance(tab["testcases"], list)
         tab["testcases"] = translate_testcases(
-            tab["testcases"], language, translation_stack
+            tab["testcases"], language, translation_stack, env
         )
     else:
         assert "scripts" in tab
         assert isinstance(tab["scripts"], list)
         tab["scripts"] = translate_testcases(
-            tab["scripts"], language, translation_stack
+            tab["scripts"], language, translation_stack, env
         )
     return tab
 
 
-def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> list:
+def translate_tabs(
+    dsl_list: list, language: str, env: Environment, translation_stack=None
+) -> list:
     if translation_stack is None:
         translation_stack = []
 
@@ -300,33 +318,48 @@ def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> lis
     for tab in dsl_list:
         assert isinstance(tab, dict)
 
-        if "translation" in tab:
-            translation_stack.append(tab["translation"])
+        if "translations" in tab:
+            translation_stack.append(tab["translations"])
 
-        result.append(translate_tab(tab, language, translation_stack))
-        if "translation" in tab:
+        result.append(translate_tab(tab, language, translation_stack, env))
+        if "translations" in tab:
             translation_stack.pop()
-            tab.pop("translation")
+            tab.pop("translations")
 
     return result
 
 
+def wrap_in_braces(value):
+    return f"{{{value}}}"
+
+
+def create_enviroment() -> Environment:
+    enviroment = Environment()
+    enviroment.filters["braces"] = wrap_in_braces
+    return enviroment
+
+
 def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
+
+    env = create_enviroment()
+
     if isinstance(dsl_object, list):
-        return translate_tabs(dsl_object, language)
+        return translate_tabs(dsl_object, language, env)
     else:
         assert isinstance(dsl_object, dict)
         key_to_set = "units" if "units" in dsl_object else "tabs"
         tab_list = dsl_object.get(key_to_set)
         assert isinstance(tab_list, list)
         translation_stack = []
-        if "translation" in dsl_object:
-            translation_stack.append(dsl_object["translation"])
-            dsl_object.pop("translation")
+        if "translations" in dsl_object:
+            translation_stack.append(dsl_object["translations"])
+            dsl_object.pop("translations")
 
         flat_stack = flatten_stack(translation_stack, language)
-        dsl_object = translate_input_files(dsl_object, language, flat_stack)
-        dsl_object[key_to_set] = translate_tabs(tab_list, language, translation_stack)
+        dsl_object = translate_input_files(dsl_object, language, flat_stack, env)
+        dsl_object[key_to_set] = translate_tabs(
+            tab_list, language, env, translation_stack
+        )
         return dsl_object
 
 
@@ -352,9 +385,13 @@ def oracle_representer(dumper, data):
     def expression_representer(dumper, data):
         return dumper.represent_scalar("!expression", data)
 
+    # def represent_str(dumper, data):
+    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
+
     # Register the representer for the ReturnOracle object
     yaml.add_representer(ReturnOracle, oracle_representer)
     yaml.add_representer(ExpressionString, expression_representer)
+    # yaml.add_representer(str, represent_str)
     return yaml.dump(yaml_object, sort_keys=False)
 
 
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 14034b8a..02e44b2a 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -23,6 +23,7 @@
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
 from tested.nat_translation import (
     convert_to_yaml,
+    create_enviroment,
     parse_value,
     translate_dsl,
     validate_pre_dsl,
@@ -1331,7 +1332,7 @@ def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
     yaml_str = """
-translation:
+translations:
   animal:
     en: "animals"
     nl: "dieren"
@@ -1341,23 +1342,26 @@ def test_natural_translate_unit_test():
   elf:
     en: "eleven"
     nl: "elf"
+  select:
+    en: "select"
+    nl: "selecteer"
 tabs:
-  - tab: "{{{animal}}}_{{{result}}}"
-    translation:
+  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}_{{ negentien|default('{{ negentien }}') }}"
+    translations:
       animal:
         en: "animal_tab"
         nl: "dier_tab"
     contexts:
       - testcases:
         - statement: !natural_language
-            en: '{result} = Trying(10)'
-            nl: '{result} = Proberen(10)'
+            en: '{{result}} = Trying(10)'
+            nl: '{{result}} = Proberen(10)'
         - expression: !natural_language
-            en: 'count_words({result})'
-            nl: 'tel_woorden({result})'
+            en: 'count_words({{result}})'
+            nl: 'tel_woorden({{result}})'
           return: !natural_language
-            en: 'The {result} is 10'
-            nl: 'Het {result} is 10'
+            en: 'The {{result}} is 10'
+            nl: 'Het {{result}} is 10'
         - expression: !natural_language
             en: !expression "count"
             nl: !expression "tellen"
@@ -1367,14 +1371,14 @@ def test_natural_translate_unit_test():
         - expression: 'ok(10)'
           return: !oracle
             value: !natural_language
-              en: "The {result} 10 is OK!"
-              nl: "Het {result} 10 is OK!"
+              en: "The {{result}} 10 is OK!"
+              nl: "Het {{result}} 10 is OK!"
             oracle: "custom_check"
             file: "test.py"
             name: "evaluate_test"
             arguments: !natural_language
               en: ["The value", "is OK!", "is not OK!"]
-              nl: ["Het {result}", "is OK!", "is niet OK!"]
+              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
           description: !natural_language
             en: "Ten"
             nl: "Tien"
@@ -1385,7 +1389,7 @@ def test_natural_translate_unit_test():
           nl:
             - name: "fileNL.txt"
               url: "media/workdir/fileNL.txt"
-        translation:
+        translations:
           result:
             en: "results_context"
             nl: "resultaten_context"
@@ -1397,33 +1401,37 @@ def test_natural_translate_unit_test():
             return: '11_{elf}'
             description:
               description: !natural_language
-                en: "Eleven_{elf}"
-                nl: "Elf_{elf}"
+                en: "Eleven_{{elf}}"
+                nl: "Elf_{{elf}}"
               format: "code"
-  - tab: '{animal}'
+  - tab: '{{animal}}'
     testcases:
       - expression: !natural_language
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
       - expression: !programming_language
-          javascript: "{animal}_javascript(1 + 1)"
-          typescript: "{animal}_typescript(1 + 1)"
-          java: "Submission.{animal}_java(1 + 1)"
+          javascript: "{{animal}}_javascript(1 + 1)"
+          typescript: "{{animal}}_typescript(1 + 1)"
+          java: "Submission.{{animal}}_java(1 + 1)"
           python: !natural_language
-            en: "{animal}_python_en(1 + 1)"
-            nl: "{animal}_python_nl(1 + 1)"
+            en: "{{animal}}_python_en(1 + 1)"
+            nl: "{{animal}}_python_nl(1 + 1)"
         return: 2
+  - tab: 'test'
+    testcases:
+      - expression: "{{select}}('a', {'a': 1, 'b': 2})"
+        return: 1
 """.strip()
     translated_yaml_str = """
 tabs:
-- tab: '{animal_tab}_{results}'
+- tab: '{animal_tab}_{results}_{{ negentien }}'
   contexts:
   - testcases:
     - statement: results_context = Trying(10)
     - expression: count_words(results_context)
       return: The results_context is 10
-    - expression: !expression 'count'
+    - expression: count
       return: count
     - expression: ok(10)
       return: !oracle
@@ -1442,7 +1450,7 @@ def test_natural_translate_unit_test():
   - testcases:
     - statement: result = Trying(11)
     - expression: result
-      return: 11_eleven
+      return: 11_{elf}
       description:
         description: Eleven_eleven
         format: code
@@ -1456,6 +1464,10 @@ def test_natural_translate_unit_test():
       java: Submission.animals_java(1 + 1)
       python: animals_python_en(1 + 1)
     return: 2
+- tab: test
+  testcases:
+  - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
+    return: 1
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")
@@ -1472,49 +1484,49 @@ def test_natural_translate_io_test():
   - unit: !natural_language
       en: "Arguments"
       nl: "Argumenten"
-    translation:
+    translations:
       User:
         en: "user"
         nl: "gebruiker"
     cases:
       - script:
         - stdin: !natural_language
-            en: "User_{User}"
-            nl: "Gebruiker_{User}"
+            en: "User_{{User}}"
+            nl: "Gebruiker_{{User}}"
           arguments: !natural_language
-            en: [ "input_{User}", "output_{User}" ]
-            nl: [ "invoer_{User}", "uitvoer_{User}" ]
+            en: [ "input_{{User}}", "output_{{User}}" ]
+            nl: [ "invoer_{{User}}", "uitvoer_{{User}}" ]
           stdout: !natural_language
-            en: "Hi {User}"
-            nl: "Hallo {User}"
+            en: "Hi {{User}}"
+            nl: "Hallo {{User}}"
           stderr: !natural_language
-            en: "Nothing to see here {User}"
-            nl: "Hier is niets te zien {User}"
+            en: "Nothing to see here {{User}}"
+            nl: "Hier is niets te zien {{User}}"
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
         - stdin: !natural_language
-            en: "Friend of {User}"
-            nl: "Vriend van {User}"
+            en: "Friend of {{User}}"
+            nl: "Vriend van {{User}}"
           arguments: !natural_language
             en: [ "input", "output" ]
             nl: [ "invoer", "uitvoer" ]
           stdout:
             data: !natural_language
-              en: "Hi Friend of {User}"
-              nl: "Hallo Vriend van {User}"
+              en: "Hi Friend of {{User}}"
+              nl: "Hallo Vriend van {{User}}"
             config:
               ignoreWhitespace: true
           stderr:
             data: !natural_language
-              en: "Nothing to see here {User}"
-              nl: "Hier is niets te zien {User}"
+              en: "Nothing to see here {{User}}"
+              nl: "Hier is niets te zien {{User}}"
             config:
               ignoreWhitespace: true
           exception:
             message: !natural_language
-              en: "Does not look good {User}"
-              nl: "Ziet er niet goed uit {User}"
+              en: "Does not look good {{User}}"
+              nl: "Ziet er niet goed uit {{User}}"
             types:
               typescript: "ERROR"
   - unit: "test"
@@ -1565,10 +1577,11 @@ def test_natural_translate_io_test():
 
 
 def test_translate_parse():
+    env = create_enviroment()
     flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
     value = {
-        "key1": ["value1_{animal}", "value1_{human}"],
-        "key2": "value2_{number}",
+        "key1": ["value1_{{animal}}", "value1_{{human}}"],
+        "key2": "value2_{{number}}",
         "key3": 10,
     }
     expected_value = {
@@ -1576,7 +1589,7 @@ def test_translate_parse():
         "key2": "value2_getal",
         "key3": 10,
     }
-    parsed_result = parse_value(value, flattened_stack)
+    parsed_result = parse_value(value, flattened_stack, env)
     assert parsed_result == expected_value
 
 

From 88a2ede5b970d0d802438132d5dd7851ce2088f0 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 4 Jan 2025 17:13:04 +0100
Subject: [PATCH 032/111] small cleanup

---
 tested/nat_translation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index f555976b..8efa4efb 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,7 +2,7 @@
 from pathlib import Path
 
 import yaml
-from jinja2 import Environment, Template
+from jinja2 import Environment
 
 from tested.dsl.translate_parser import (
     DslValidationError,

From e6e548ca20b7d6cb6239be6fced509a7b21a34e9 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 31 Jan 2025 13:08:46 +0100
Subject: [PATCH 033/111] moved tests to new file

---
 tests/test_dsl_yaml.py       | 288 ----------------------------------
 tests/test_preprocess_dsl.py | 293 +++++++++++++++++++++++++++++++++++
 2 files changed, 293 insertions(+), 288 deletions(-)
 create mode 100644 tests/test_preprocess_dsl.py

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 02e44b2a..5212567f 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1328,291 +1328,3 @@ def test_editor_json_schema_is_valid():
     validator.check_schema(validator.schema)
 
 
-def test_natural_translate_unit_test():
-    # Everywhere where !natural_language is used, it is mandatory to do so.
-    # Everywhere else it isn't.
-    yaml_str = """
-translations:
-  animal:
-    en: "animals"
-    nl: "dieren"
-  result:
-    en: "results"
-    nl: "resultaten"
-  elf:
-    en: "eleven"
-    nl: "elf"
-  select:
-    en: "select"
-    nl: "selecteer"
-tabs:
-  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}_{{ negentien|default('{{ negentien }}') }}"
-    translations:
-      animal:
-        en: "animal_tab"
-        nl: "dier_tab"
-    contexts:
-      - testcases:
-        - statement: !natural_language
-            en: '{{result}} = Trying(10)'
-            nl: '{{result}} = Proberen(10)'
-        - expression: !natural_language
-            en: 'count_words({{result}})'
-            nl: 'tel_woorden({{result}})'
-          return: !natural_language
-            en: 'The {{result}} is 10'
-            nl: 'Het {{result}} is 10'
-        - expression: !natural_language
-            en: !expression "count"
-            nl: !expression "tellen"
-          return: !natural_language
-            en: 'count'
-            nl: 'tellen'
-        - expression: 'ok(10)'
-          return: !oracle
-            value: !natural_language
-              en: "The {{result}} 10 is OK!"
-              nl: "Het {{result}} 10 is OK!"
-            oracle: "custom_check"
-            file: "test.py"
-            name: "evaluate_test"
-            arguments: !natural_language
-              en: ["The value", "is OK!", "is not OK!"]
-              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
-          description: !natural_language
-            en: "Ten"
-            nl: "Tien"
-        files: !natural_language
-          en:
-            - name: "file.txt"
-              url: "media/workdir/file.txt"
-          nl:
-            - name: "fileNL.txt"
-              url: "media/workdir/fileNL.txt"
-        translations:
-          result:
-            en: "results_context"
-            nl: "resultaten_context"
-      - testcases:
-          - statement: !natural_language
-              en: 'result = Trying(11)'
-              nl: 'resultaat = Proberen(11)'
-          - expression: 'result'
-            return: '11_{elf}'
-            description:
-              description: !natural_language
-                en: "Eleven_{{elf}}"
-                nl: "Elf_{{elf}}"
-              format: "code"
-  - tab: '{{animal}}'
-    testcases:
-      - expression: !natural_language
-          en: "tests(11)"
-          nl: "testen(11)"
-        return: 11
-      - expression: !programming_language
-          javascript: "{{animal}}_javascript(1 + 1)"
-          typescript: "{{animal}}_typescript(1 + 1)"
-          java: "Submission.{{animal}}_java(1 + 1)"
-          python: !natural_language
-            en: "{{animal}}_python_en(1 + 1)"
-            nl: "{{animal}}_python_nl(1 + 1)"
-        return: 2
-  - tab: 'test'
-    testcases:
-      - expression: "{{select}}('a', {'a': 1, 'b': 2})"
-        return: 1
-""".strip()
-    translated_yaml_str = """
-tabs:
-- tab: '{animal_tab}_{results}_{{ negentien }}'
-  contexts:
-  - testcases:
-    - statement: results_context = Trying(10)
-    - expression: count_words(results_context)
-      return: The results_context is 10
-    - expression: count
-      return: count
-    - expression: ok(10)
-      return: !oracle
-        value: The results_context 10 is OK!
-        oracle: custom_check
-        file: test.py
-        name: evaluate_test
-        arguments:
-        - The value
-        - is OK!
-        - is not OK!
-      description: Ten
-    files:
-    - name: file.txt
-      url: media/workdir/file.txt
-  - testcases:
-    - statement: result = Trying(11)
-    - expression: result
-      return: 11_{elf}
-      description:
-        description: Eleven_eleven
-        format: code
-- tab: animals
-  testcases:
-  - expression: tests(11)
-    return: 11
-  - expression:
-      javascript: animals_javascript(1 + 1)
-      typescript: animals_typescript(1 + 1)
-      java: Submission.animals_java(1 + 1)
-      python: animals_python_en(1 + 1)
-    return: 2
-- tab: test
-  testcases:
-  - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
-    return: 1
-""".strip()
-    parsed_yaml = _parse_yaml(yaml_str)
-    translated_dsl = translate_dsl(parsed_yaml, "en")
-    translated_yaml = convert_to_yaml(translated_dsl)
-    print(translated_yaml)
-    assert translated_yaml.strip() == translated_yaml_str
-
-
-def test_natural_translate_io_test():
-    # Everywhere where !natural_language is used, it is mandatory to do so.
-    # Everywhere else it isn't.
-    yaml_str = """
-units:
-  - unit: !natural_language
-      en: "Arguments"
-      nl: "Argumenten"
-    translations:
-      User:
-        en: "user"
-        nl: "gebruiker"
-    cases:
-      - script:
-        - stdin: !natural_language
-            en: "User_{{User}}"
-            nl: "Gebruiker_{{User}}"
-          arguments: !natural_language
-            en: [ "input_{{User}}", "output_{{User}}" ]
-            nl: [ "invoer_{{User}}", "uitvoer_{{User}}" ]
-          stdout: !natural_language
-            en: "Hi {{User}}"
-            nl: "Hallo {{User}}"
-          stderr: !natural_language
-            en: "Nothing to see here {{User}}"
-            nl: "Hier is niets te zien {{User}}"
-          exception: !natural_language
-            en: "Does not look good"
-            nl: "Ziet er niet goed uit"
-        - stdin: !natural_language
-            en: "Friend of {{User}}"
-            nl: "Vriend van {{User}}"
-          arguments: !natural_language
-            en: [ "input", "output" ]
-            nl: [ "invoer", "uitvoer" ]
-          stdout:
-            data: !natural_language
-              en: "Hi Friend of {{User}}"
-              nl: "Hallo Vriend van {{User}}"
-            config:
-              ignoreWhitespace: true
-          stderr:
-            data: !natural_language
-              en: "Nothing to see here {{User}}"
-              nl: "Hier is niets te zien {{User}}"
-            config:
-              ignoreWhitespace: true
-          exception:
-            message: !natural_language
-              en: "Does not look good {{User}}"
-              nl: "Ziet er niet goed uit {{User}}"
-            types:
-              typescript: "ERROR"
-  - unit: "test"
-    scripts:
-      - expression: !natural_language
-            en: "tests(11)"
-            nl: "testen(11)"
-        return: 11
-""".strip()
-    translated_yaml_str = """
-units:
-- unit: Arguments
-  cases:
-  - script:
-    - stdin: User_user
-      arguments:
-      - input_user
-      - output_user
-      stdout: Hi user
-      stderr: Nothing to see here user
-      exception: Does not look good
-    - stdin: Friend of user
-      arguments:
-      - input
-      - output
-      stdout:
-        data: Hi Friend of user
-        config:
-          ignoreWhitespace: true
-      stderr:
-        data: Nothing to see here user
-        config:
-          ignoreWhitespace: true
-      exception:
-        message: Does not look good user
-        types:
-          typescript: ERROR
-- unit: test
-  scripts:
-  - expression: tests(11)
-    return: 11
-""".strip()
-    parsed_yaml = _parse_yaml(yaml_str)
-    translated_dsl = translate_dsl(parsed_yaml, "en")
-    translated_yaml = convert_to_yaml(translated_dsl)
-    print(translated_yaml)
-    assert translated_yaml.strip() == translated_yaml_str
-
-
-def test_translate_parse():
-    env = create_enviroment()
-    flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
-    value = {
-        "key1": ["value1_{{animal}}", "value1_{{human}}"],
-        "key2": "value2_{{number}}",
-        "key3": 10,
-    }
-    expected_value = {
-        "key1": ["value1_dier", "value1_mens"],
-        "key2": "value2_getal",
-        "key3": 10,
-    }
-    parsed_result = parse_value(value, flattened_stack, env)
-    assert parsed_result == expected_value
-
-
-def test_wrong_natural_translation_suite():
-    yaml_str = """
-tabs:
-- tab: animals
-  testcases:
-  - expression: tests(11)
-    return: 11
-  - expression:
-      javascript: animals_javascript(1 + 1)
-      typescript: animals_typescript(1 + 1)
-      java: Submission.animals_java(1 + 1)
-      python:
-        en: animals_python_en(1 + 1)
-        nl: animals_python_nl(1 + 1)
-    return: 2
-    """.strip()
-    parsed_yaml = _parse_yaml(yaml_str)
-    try:
-        validate_pre_dsl(parsed_yaml)
-    except ExceptionGroup:
-        print("As expected")
-    else:
-        assert False, "Expected ExceptionGroup, but no exception was raised"
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
new file mode 100644
index 00000000..efd684d2
--- /dev/null
+++ b/tests/test_preprocess_dsl.py
@@ -0,0 +1,293 @@
+from tested.dsl.translate_parser import _parse_yaml
+from tested.nat_translation import validate_pre_dsl, translate_dsl, convert_to_yaml, \
+    create_enviroment, parse_value
+
+
+def test_natural_translate_unit_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
+    yaml_str = """
+translations:
+  animal:
+    en: "animals"
+    nl: "dieren"
+  result:
+    en: "results"
+    nl: "resultaten"
+  elf:
+    en: "eleven"
+    nl: "elf"
+  select:
+    en: "select"
+    nl: "selecteer"
+tabs:
+  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}_{{ negentien|default('{{ negentien }}') }}"
+    translations:
+      animal:
+        en: "animal_tab"
+        nl: "dier_tab"
+    contexts:
+      - testcases:
+        - statement: !natural_language
+            en: '{{result}} = Trying(10)'
+            nl: '{{result}} = Proberen(10)'
+        - expression: !natural_language
+            en: 'count_words({{result}})'
+            nl: 'tel_woorden({{result}})'
+          return: !natural_language
+            en: 'The {{result}} is 10'
+            nl: 'Het {{result}} is 10'
+        - expression: !natural_language
+            en: !expression "count"
+            nl: !expression "tellen"
+          return: !natural_language
+            en: 'count'
+            nl: 'tellen'
+        - expression: 'ok(10)'
+          return: !oracle
+            value: !natural_language
+              en: "The {{result}} 10 is OK!"
+              nl: "Het {{result}} 10 is OK!"
+            oracle: "custom_check"
+            file: "test.py"
+            name: "evaluate_test"
+            arguments: !natural_language
+              en: ["The value", "is OK!", "is not OK!"]
+              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
+          description: !natural_language
+            en: "Ten"
+            nl: "Tien"
+        files: !natural_language
+          en:
+            - name: "file.txt"
+              url: "media/workdir/file.txt"
+          nl:
+            - name: "fileNL.txt"
+              url: "media/workdir/fileNL.txt"
+        translations:
+          result:
+            en: "results_context"
+            nl: "resultaten_context"
+      - testcases:
+          - statement: !natural_language
+              en: 'result = Trying(11)'
+              nl: 'resultaat = Proberen(11)'
+          - expression: 'result'
+            return: '11_{elf}'
+            description:
+              description: !natural_language
+                en: "Eleven_{{elf}}"
+                nl: "Elf_{{elf}}"
+              format: "code"
+  - tab: '{{animal}}'
+    testcases:
+      - expression: !natural_language
+          en: "tests(11)"
+          nl: "testen(11)"
+        return: 11
+      - expression: !programming_language
+          javascript: "{{animal}}_javascript(1 + 1)"
+          typescript: "{{animal}}_typescript(1 + 1)"
+          java: "Submission.{{animal}}_java(1 + 1)"
+          python: !natural_language
+            en: "{{animal}}_python_en(1 + 1)"
+            nl: "{{animal}}_python_nl(1 + 1)"
+        return: 2
+  - tab: 'test'
+    testcases:
+      - expression: "{{select}}('a', {'a': 1, 'b': 2})"
+        return: 1
+""".strip()
+    translated_yaml_str = """
+tabs:
+- tab: '{animal_tab}_{results}_{{ negentien }}'
+  contexts:
+  - testcases:
+    - statement: results_context = Trying(10)
+    - expression: count_words(results_context)
+      return: The results_context is 10
+    - expression: count
+      return: count
+    - expression: ok(10)
+      return: !oracle
+        value: The results_context 10 is OK!
+        oracle: custom_check
+        file: test.py
+        name: evaluate_test
+        arguments:
+        - The value
+        - is OK!
+        - is not OK!
+      description: Ten
+    files:
+    - name: file.txt
+      url: media/workdir/file.txt
+  - testcases:
+    - statement: result = Trying(11)
+    - expression: result
+      return: 11_{elf}
+      description:
+        description: Eleven_eleven
+        format: code
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python: animals_python_en(1 + 1)
+    return: 2
+- tab: test
+  testcases:
+  - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
+    return: 1
+""".strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml.strip() == translated_yaml_str
+
+
+def test_natural_translate_io_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
+    yaml_str = """
+units:
+  - unit: !natural_language
+      en: "Arguments"
+      nl: "Argumenten"
+    translations:
+      User:
+        en: "user"
+        nl: "gebruiker"
+    cases:
+      - script:
+        - stdin: !natural_language
+            en: "User_{{User}}"
+            nl: "Gebruiker_{{User}}"
+          arguments: !natural_language
+            en: [ "input_{{User}}", "output_{{User}}" ]
+            nl: [ "invoer_{{User}}", "uitvoer_{{User}}" ]
+          stdout: !natural_language
+            en: "Hi {{User}}"
+            nl: "Hallo {{User}}"
+          stderr: !natural_language
+            en: "Nothing to see here {{User}}"
+            nl: "Hier is niets te zien {{User}}"
+          exception: !natural_language
+            en: "Does not look good"
+            nl: "Ziet er niet goed uit"
+        - stdin: !natural_language
+            en: "Friend of {{User}}"
+            nl: "Vriend van {{User}}"
+          arguments: !natural_language
+            en: [ "input", "output" ]
+            nl: [ "invoer", "uitvoer" ]
+          stdout:
+            data: !natural_language
+              en: "Hi Friend of {{User}}"
+              nl: "Hallo Vriend van {{User}}"
+            config:
+              ignoreWhitespace: true
+          stderr:
+            data: !natural_language
+              en: "Nothing to see here {{User}}"
+              nl: "Hier is niets te zien {{User}}"
+            config:
+              ignoreWhitespace: true
+          exception:
+            message: !natural_language
+              en: "Does not look good {{User}}"
+              nl: "Ziet er niet goed uit {{User}}"
+            types:
+              typescript: "ERROR"
+  - unit: "test"
+    scripts:
+      - expression: !natural_language
+            en: "tests(11)"
+            nl: "testen(11)"
+        return: 11
+""".strip()
+    translated_yaml_str = """
+units:
+- unit: Arguments
+  cases:
+  - script:
+    - stdin: User_user
+      arguments:
+      - input_user
+      - output_user
+      stdout: Hi user
+      stderr: Nothing to see here user
+      exception: Does not look good
+    - stdin: Friend of user
+      arguments:
+      - input
+      - output
+      stdout:
+        data: Hi Friend of user
+        config:
+          ignoreWhitespace: true
+      stderr:
+        data: Nothing to see here user
+        config:
+          ignoreWhitespace: true
+      exception:
+        message: Does not look good user
+        types:
+          typescript: ERROR
+- unit: test
+  scripts:
+  - expression: tests(11)
+    return: 11
+""".strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml.strip() == translated_yaml_str
+
+
+def test_translate_parse():
+    env = create_enviroment()
+    flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
+    value = {
+        "key1": ["value1_{{animal}}", "value1_{{human}}"],
+        "key2": "value2_{{number}}",
+        "key3": 10,
+    }
+    expected_value = {
+        "key1": ["value1_dier", "value1_mens"],
+        "key2": "value2_getal",
+        "key3": 10,
+    }
+    parsed_result = parse_value(value, flattened_stack, env)
+    assert parsed_result == expected_value
+
+
+def test_wrong_natural_translation_suite():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python:
+        en: animals_python_en(1 + 1)
+        nl: animals_python_nl(1 + 1)
+    return: 2
+    """.strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    try:
+        validate_pre_dsl(parsed_yaml)
+    except ExceptionGroup:
+        print("As expected")
+    else:
+        assert False, "Expected ExceptionGroup, but no exception was raised"

From 95f85241d5fc6ac972c9766f365afb795bc89fa3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 31 Jan 2025 13:13:11 +0100
Subject: [PATCH 034/111] Small cleanup

---
 tests/test_dsl_yaml.py       | 12 ++----------
 tests/test_preprocess_dsl.py |  9 +++++++--
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 5212567f..3d60b58c 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -20,14 +20,8 @@
     StringTypes,
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
-from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import (
-    convert_to_yaml,
-    create_enviroment,
-    parse_value,
-    translate_dsl,
-    validate_pre_dsl,
-)
+from tested.dsl.translate_parser import load_schema_validator
+
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1326,5 +1320,3 @@ def test_editor_json_schema_is_valid():
     validator = load_schema_validator("schema.json")
     assert isinstance(validator.schema, dict)
     validator.check_schema(validator.schema)
-
-
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index efd684d2..a6cd8501 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,6 +1,11 @@
 from tested.dsl.translate_parser import _parse_yaml
-from tested.nat_translation import validate_pre_dsl, translate_dsl, convert_to_yaml, \
-    create_enviroment, parse_value
+from tested.nat_translation import (
+    convert_to_yaml,
+    create_enviroment,
+    parse_value,
+    translate_dsl,
+    validate_pre_dsl,
+)
 
 
 def test_natural_translate_unit_test():

From 9ac66cbcf8a19f3b4c500bc7c906387e7a8aadff Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 31 Jan 2025 13:19:26 +0100
Subject: [PATCH 035/111] fix isort

---
 tests/test_dsl_yaml.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 3d60b58c..3caef72d 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,6 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import load_schema_validator
-
 from tested.serialisation import (
     FunctionCall,
     NumberType,

From b7012d3518947fc1bb5f7eed5aa17acdc4f73a55 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 18 Feb 2025 19:50:37 +0100
Subject: [PATCH 036/111] got rid of usage of instanceof

---
 tested/dsl/translate_parser.py | 42 +++++++++++++++++----
 tested/nat_translation.py      | 68 +++++++++++++++++-----------------
 2 files changed, 67 insertions(+), 43 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 1c0d1e5d..0f30d9c0 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -80,21 +80,40 @@ class TestedType:
     type: str | AllTypes
 
 
-class ExpressionString(str):
-    pass
+# class ExpressionString(str):
+#     pass
+#
+#
+# class ReturnOracle(dict):
+#     pass
+#
+#
+# class NaturalLanguageMap(dict):
+#     pass
+#
+#
+# class ProgrammingLanguageMap(dict):
+#     pass
 
+class ExpressionString(str):
+    @staticmethod
+    def get_type_name() -> str:
+        return "ExpressionString"
 
 class ReturnOracle(dict):
-    pass
-
+    @staticmethod
+    def get_type_name() -> str:
+        return "ReturnOracle"
 
 class NaturalLanguageMap(dict):
-    pass
-
+    @staticmethod
+    def get_type_name() -> str:
+        return "NaturalLanguageMap"
 
 class ProgrammingLanguageMap(dict):
-    pass
-
+    @staticmethod
+    def get_type_name() -> str:
+        return "ProgrammingLanguageMap"
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
@@ -112,6 +131,13 @@ class ProgrammingLanguageMap(dict):
 )
 
 
+# Function that calls translate if available
+def visit_yaml_object(obj: YamlObject, type:str) -> bool:
+    if hasattr(obj, "get_type_name"):
+        return obj.get_type_name() == type
+    return obj.__class__.__name__ == type
+
+
 def _convert_language_dictionary(
     original: dict[str, str]
 ) -> dict[SupportedLanguage, str]:
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 8efa4efb..03deea46 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,7 +16,7 @@
     _validate_dsl,
     _validate_testcase_combinations,
     convert_validation_error_to_group,
-    load_schema_validator,
+    load_schema_validator, visit_yaml_object,
 )
 
 
@@ -46,7 +46,7 @@ def validate_pre_dsl(dsl_object: YamlObject):
 
 
 def natural_language_map_translation(value: YamlObject, language: str):
-    if isinstance(value, NaturalLanguageMap):
+    if visit_yaml_object(value, "NaturalLanguageMap"):
         assert language in value
         value = value[language]
     return value
@@ -58,17 +58,17 @@ def translate_input_files(
     if (files := dsl_object.get("files")) is not None:
         # Translation map can happen at the top level.
         files = natural_language_map_translation(files, language)
-        assert isinstance(files, list)
+        assert visit_yaml_object(files, "list")
         for i in range(len(files)):
             file = files[i]
 
             # Do the formatting.
-            if isinstance(file, dict):
+            if visit_yaml_object(file, "dict"):
                 name = file["name"]
-                assert isinstance(name, str)
+                assert visit_yaml_object(name, "str")
                 file["name"] = format_string(name, flattened_stack, env)
                 url = file["url"]
-                assert isinstance(url, str)
+                assert visit_yaml_object(url, "str")
                 file["url"] = format_string(url, flattened_stack, env)
             files[i] = file
 
@@ -81,11 +81,11 @@ def parse_value(
 ) -> YamlObject:
 
     # Will format the strings in different values.
-    if isinstance(value, str):
+    if visit_yaml_object(value, "str"):
         return format_string(value, flattened_stack, env)
-    elif isinstance(value, dict):
+    elif visit_yaml_object(value, "dict"):
         return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
-    elif isinstance(value, list):
+    elif visit_yaml_object(value, "list"):
         return [parse_value(v, flattened_stack, env) for v in value]
 
     return value
@@ -117,12 +117,12 @@ def translate_io(
     # Translate NaturalLanguageMap
     io_object = natural_language_map_translation(io_object, language)
 
-    if isinstance(io_object, dict):
+    if visit_yaml_object(io_object, "dict"):
         data = natural_language_map_translation(io_object[key], language)
         io_object[key] = parse_value(data, flat_stack, env)
 
     # Perform translation based of translation stack.
-    elif isinstance(io_object, str):
+    elif visit_yaml_object(io_object, "str"):
         return format_string(io_object, flat_stack, env)
 
     return io_object
@@ -137,14 +137,12 @@ def translate_testcase(
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
         # Program language translation found
-        if isinstance(expr_stmt, ProgrammingLanguageMap):
+        if visit_yaml_object(expr_stmt, "ProgrammingLanguageMap"):
             expr_stmt = {
                 k: natural_language_map_translation(v, language)
                 for k, v in expr_stmt.items()
             }
-        elif isinstance(
-            expr_stmt, NaturalLanguageMap
-        ):  # Natural language translation found
+        elif visit_yaml_object(expr_stmt, "NaturalLanguageMap"):
             assert language in expr_stmt
             expr_stmt = expr_stmt[language]
 
@@ -155,7 +153,7 @@ def translate_testcase(
             stdin_stmt = natural_language_map_translation(stdin_stmt, language)
 
             # Perform translation based of translation stack.
-            assert isinstance(stdin_stmt, str)
+            assert visit_yaml_object(stdin_stmt, "str")
             testcase["stdin"] = format_string(stdin_stmt, flat_stack, env)
 
         # Translate NaturalLanguageMap
@@ -163,7 +161,7 @@ def translate_testcase(
         arguments = natural_language_map_translation(arguments, language)
 
         # Perform translation based of translation stack.
-        assert isinstance(arguments, list)
+        assert visit_yaml_object(arguments, "list")
         testcase["arguments"] = parse_value(arguments, flat_stack, env)
 
     if (stdout := testcase.get("stdout")) is not None:
@@ -173,7 +171,7 @@ def translate_testcase(
         # Translate NaturalLanguageMap
         file = natural_language_map_translation(file, language)
 
-        assert isinstance(file, dict)
+        assert visit_yaml_object(file, "dict")
         file["content"] = format_string(str(file["content"]), flat_stack, env)
         file["location"] = format_string(str(file["location"]), flat_stack, env)
 
@@ -188,7 +186,7 @@ def translate_testcase(
         )
 
     if (result := testcase.get("return")) is not None:
-        if isinstance(result, ReturnOracle):
+        if visit_yaml_object(result, "ReturnOracle"):
             arguments = result.get("arguments", [])
             arguments = natural_language_map_translation(arguments, language)
 
@@ -201,7 +199,7 @@ def translate_testcase(
             result["value"] = parse_value(value, flat_stack, env)
             testcase["return"] = result
 
-        elif isinstance(result, NaturalLanguageMap):
+        elif visit_yaml_object(result, "NaturalLanguageMap"):
             assert language in result
             testcase["return"] = parse_value(result[language], flat_stack, env)
         elif result is not None:
@@ -210,14 +208,14 @@ def translate_testcase(
     if (description := testcase.get("description")) is not None:
         description = natural_language_map_translation(description, language)
 
-        if isinstance(description, str):
+        if visit_yaml_object(description, "str"):
             testcase["description"] = format_string(description, flat_stack, env)
         else:
-            assert isinstance(description, dict)
+            assert visit_yaml_object(description, "dict")
             dd = description["description"]
             dd = natural_language_map_translation(dd, language)
 
-            assert isinstance(dd, str)
+            assert visit_yaml_object(dd, "str")
             description["description"] = format_string(dd, flat_stack, env)
 
     testcase = translate_input_files(testcase, language, flat_stack, env)
@@ -230,7 +228,7 @@ def translate_testcases(
 ) -> list:
     result = []
     for testcase in testcases:
-        assert isinstance(testcase, dict)
+        assert visit_yaml_object(testcase, "dict")
         result.append(translate_testcase(testcase, language, translation_stack, env))
 
     return result
@@ -241,7 +239,7 @@ def translate_contexts(
 ) -> list:
     result = []
     for context in contexts:
-        assert isinstance(context, dict)
+        assert visit_yaml_object(context, "dict")
 
         # Add translation to stack
         if "translations" in context:
@@ -249,7 +247,7 @@ def translate_contexts(
 
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
-        assert isinstance(raw_testcases, list)
+        assert visit_yaml_object(raw_testcases, "list")
         context[key_to_set] = translate_testcases(
             raw_testcases, language, translation_stack, env
         )
@@ -273,7 +271,7 @@ def translate_tab(
     name = tab.get(key_to_set)
     name = natural_language_map_translation(name, language)
 
-    assert isinstance(name, str)
+    assert visit_yaml_object(name, "str")
     flat_stack = flatten_stack(translation_stack, language)
     tab[key_to_set] = format_string(name, flat_stack, env)
 
@@ -281,27 +279,27 @@ def translate_tab(
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:
-        assert isinstance(tab["contexts"], list)
+        assert visit_yaml_object(tab["contexts"], "list")
         tab["contexts"] = translate_contexts(
             tab["contexts"], language, translation_stack, env
         )
     elif "cases" in tab:
         assert "unit" in tab
         # We have testcases N.S. / contexts O.S.
-        assert isinstance(tab["cases"], list)
+        assert visit_yaml_object(tab["cases"], "list")
         tab["cases"] = translate_contexts(
             tab["cases"], language, translation_stack, env
         )
     elif "testcases" in tab:
         # We have scripts N.S. / testcases O.S.
         assert "tab" in tab
-        assert isinstance(tab["testcases"], list)
+        assert visit_yaml_object(tab["testcases"], "list")
         tab["testcases"] = translate_testcases(
             tab["testcases"], language, translation_stack, env
         )
     else:
         assert "scripts" in tab
-        assert isinstance(tab["scripts"], list)
+        assert visit_yaml_object(tab["scripts"], "list")
         tab["scripts"] = translate_testcases(
             tab["scripts"], language, translation_stack, env
         )
@@ -316,7 +314,7 @@ def translate_tabs(
 
     result = []
     for tab in dsl_list:
-        assert isinstance(tab, dict)
+        assert visit_yaml_object(tab, "dict")
 
         if "translations" in tab:
             translation_stack.append(tab["translations"])
@@ -343,13 +341,13 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
 
     env = create_enviroment()
 
-    if isinstance(dsl_object, list):
+    if visit_yaml_object(dsl_object,"lists"):
         return translate_tabs(dsl_object, language, env)
     else:
-        assert isinstance(dsl_object, dict)
+        assert visit_yaml_object(dsl_object,"dict")
         key_to_set = "units" if "units" in dsl_object else "tabs"
         tab_list = dsl_object.get(key_to_set)
-        assert isinstance(tab_list, list)
+        assert visit_yaml_object(tab_list,"list")
         translation_stack = []
         if "translations" in dsl_object:
             translation_stack.append(dsl_object["translations"])

From f630a05bc968a9b497f3ee8755673854a870cf7a Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 19 Feb 2025 17:17:43 +0100
Subject: [PATCH 037/111] fixed test

---
 tested/dsl/translate_parser.py | 7 ++++++-
 tested/nat_translation.py      | 9 +++++----
 tests/test_preprocess_dsl.py   | 2 +-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 0f30d9c0..99142c70 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -95,26 +95,31 @@ class TestedType:
 # class ProgrammingLanguageMap(dict):
 #     pass
 
+
 class ExpressionString(str):
     @staticmethod
     def get_type_name() -> str:
         return "ExpressionString"
 
+
 class ReturnOracle(dict):
     @staticmethod
     def get_type_name() -> str:
         return "ReturnOracle"
 
+
 class NaturalLanguageMap(dict):
     @staticmethod
     def get_type_name() -> str:
         return "NaturalLanguageMap"
 
+
 class ProgrammingLanguageMap(dict):
     @staticmethod
     def get_type_name() -> str:
         return "ProgrammingLanguageMap"
 
+
 OptionDict = dict[str, int | bool]
 YamlObject = (
     YamlDict
@@ -132,7 +137,7 @@ def get_type_name() -> str:
 
 
 # Function that calls translate if available
-def visit_yaml_object(obj: YamlObject, type:str) -> bool:
+def visit_yaml_object(obj: YamlObject, type: str) -> bool:
     if hasattr(obj, "get_type_name"):
         return obj.get_type_name() == type
     return obj.__class__.__name__ == type
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 03deea46..5274e07b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,7 +16,8 @@
     _validate_dsl,
     _validate_testcase_combinations,
     convert_validation_error_to_group,
-    load_schema_validator, visit_yaml_object,
+    load_schema_validator,
+    visit_yaml_object,
 )
 
 
@@ -341,13 +342,13 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
 
     env = create_enviroment()
 
-    if visit_yaml_object(dsl_object,"lists"):
+    if visit_yaml_object(dsl_object, "lists"):
         return translate_tabs(dsl_object, language, env)
     else:
-        assert visit_yaml_object(dsl_object,"dict")
+        assert visit_yaml_object(dsl_object, "dict")
         key_to_set = "units" if "units" in dsl_object else "tabs"
         tab_list = dsl_object.get(key_to_set)
-        assert visit_yaml_object(tab_list,"list")
+        assert visit_yaml_object(tab_list, "list")
         translation_stack = []
         if "translations" in dsl_object:
             translation_stack.append(dsl_object["translations"])
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index a6cd8501..4c5fc1b4 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -111,7 +111,7 @@ def test_natural_translate_unit_test():
     - statement: results_context = Trying(10)
     - expression: count_words(results_context)
       return: The results_context is 10
-    - expression: count
+    - expression: !expression 'count'
       return: count
     - expression: ok(10)
       return: !oracle

From a5d7f61c930764d36e6749f2b152c197908826ec Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 19 Feb 2025 17:19:28 +0100
Subject: [PATCH 038/111] Made small variable name change

---
 tested/dsl/translate_parser.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 99142c70..d32fa715 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -137,10 +137,10 @@ def get_type_name() -> str:
 
 
 # Function that calls translate if available
-def visit_yaml_object(obj: YamlObject, type: str) -> bool:
+def visit_yaml_object(obj: YamlObject, type_name: str) -> bool:
     if hasattr(obj, "get_type_name"):
-        return obj.get_type_name() == type
-    return obj.__class__.__name__ == type
+        return obj.get_type_name() == type_name
+    return obj.__class__.__name__ == type_name
 
 
 def _convert_language_dictionary(

From 1b9775d68116e6435791b945b3c6c82781d10025 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 22 Feb 2025 16:32:43 +0100
Subject: [PATCH 039/111] rewrote the pre-processor

---
 tested/dsl/translate_parser.py |   2 -
 tested/nat_translation.py      | 532 ++++++++++++++-------------------
 2 files changed, 222 insertions(+), 312 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index d32fa715..68a9529d 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -215,8 +215,6 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
             yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
     yaml.add_constructor("!expression", _expression_string, loader)
     yaml.add_constructor("!oracle", _return_oracle, loader)
-    yaml.add_constructor("!natural_language", _natural_language_map, loader)
-    yaml.add_constructor("!programming_language", _programming_language_map, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 5274e07b..c29d27fe 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,9 +1,13 @@
 import sys
+import textwrap
+from collections import deque
 from pathlib import Path
+from typing import cast, Any
 
 import yaml
 from jinja2 import Environment
 
+from tested.datatypes import AllTypes
 from tested.dsl.translate_parser import (
     DslValidationError,
     ExpressionString,
@@ -18,63 +22,13 @@
     convert_validation_error_to_group,
     load_schema_validator,
     visit_yaml_object,
+    _custom_type_constructors,
+    _expression_string,
+    _return_oracle,
+    _natural_language_map,
+    _programming_language_map,
 )
-
-
-def validate_pre_dsl(dsl_object: YamlObject):
-    """
-    Validate a DSl object.
-
-    :param dsl_object: The object to validate.
-    :return: True if valid, False otherwise.
-    """
-    _SCHEMA_VALIDATOR = load_schema_validator("schema-strict-nat-translation.json")
-    errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
-    if len(errors) == 1:
-        message = (
-            "Validating the DSL resulted in an error. "
-            "The most specific sub-exception is often the culprit. "
-        )
-        error = convert_validation_error_to_group(errors[0])
-        if isinstance(error, ExceptionGroup):
-            raise ExceptionGroup(message, error.exceptions)
-        else:
-            raise DslValidationError(message + str(error)) from error
-    elif len(errors) > 1:
-        the_errors = [convert_validation_error_to_group(e) for e in errors]
-        message = "Validating the DSL resulted in some errors."
-        raise ExceptionGroup(message, the_errors)
-
-
-def natural_language_map_translation(value: YamlObject, language: str):
-    if visit_yaml_object(value, "NaturalLanguageMap"):
-        assert language in value
-        value = value[language]
-    return value
-
-
-def translate_input_files(
-    dsl_object: dict, language: str, flattened_stack: dict, env: Environment
-) -> dict:
-    if (files := dsl_object.get("files")) is not None:
-        # Translation map can happen at the top level.
-        files = natural_language_map_translation(files, language)
-        assert visit_yaml_object(files, "list")
-        for i in range(len(files)):
-            file = files[i]
-
-            # Do the formatting.
-            if visit_yaml_object(file, "dict"):
-                name = file["name"]
-                assert visit_yaml_object(name, "str")
-                file["name"] = format_string(name, flattened_stack, env)
-                url = file["url"]
-                assert visit_yaml_object(url, "str")
-                file["url"] = format_string(url, flattened_stack, env)
-            files[i] = file
-
-        dsl_object["files"] = files
-    return dsl_object
+from tested.utils import get_args
 
 
 def parse_value(
@@ -86,321 +40,279 @@ def parse_value(
         return format_string(value, flattened_stack, env)
     elif visit_yaml_object(value, "dict"):
         return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
-    elif visit_yaml_object(value, "list"):
+    elif visit_yaml_object(value, "list") and len(value) > 0:
         return [parse_value(v, flattened_stack, env) for v in value]
 
     return value
 
 
-def flatten_stack(translation_stack: list, language: str) -> dict:
+def flatten_stack(translation_stack: list) -> dict:
     # Will transform a list of translation maps into a dict that
     # has all the keys defined over all the different translation map and will have
     # the value of the newest definition. In this definition we also chose
     # the translation of the provided language.
     flattened = {}
     for d in translation_stack:
-        flattened.update({k: v[language] for k, v in d.items() if language in v})
+        flattened.update({k: v for k, v in d.items()})
     return flattened
 
 
 def format_string(string: str, translations: dict, env: Environment) -> str:
     template = env.from_string(string)
     result = template.render(translations)
-    # print(f"jinja result: {result}")
 
-    # return string.format(**translations)
     return result
 
 
-def translate_io(
-    io_object: YamlObject, key: str, language: str, flat_stack: dict, env: Environment
-) -> YamlObject:
-    # Translate NaturalLanguageMap
-    io_object = natural_language_map_translation(io_object, language)
-
-    if visit_yaml_object(io_object, "dict"):
-        data = natural_language_map_translation(io_object[key], language)
-        io_object[key] = parse_value(data, flat_stack, env)
-
-    # Perform translation based of translation stack.
-    elif visit_yaml_object(io_object, "str"):
-        return format_string(io_object, flat_stack, env)
-
-    return io_object
-
-
-def translate_testcase(
-    testcase: YamlDict, language: str, translation_stack: list, env: Environment
-) -> YamlDict:
-    _validate_testcase_combinations(testcase)
-    flat_stack = flatten_stack(translation_stack, language)
-
-    key_to_set = "statement" if "statement" in testcase else "expression"
-    if (expr_stmt := testcase.get(key_to_set)) is not None:
-        # Program language translation found
-        if visit_yaml_object(expr_stmt, "ProgrammingLanguageMap"):
-            expr_stmt = {
-                k: natural_language_map_translation(v, language)
-                for k, v in expr_stmt.items()
-            }
-        elif visit_yaml_object(expr_stmt, "NaturalLanguageMap"):
-            assert language in expr_stmt
-            expr_stmt = expr_stmt[language]
-
-        testcase[key_to_set] = parse_value(expr_stmt, flat_stack, env)
-    else:
-        if (stdin_stmt := testcase.get("stdin")) is not None:
-            # Translate NaturalLanguageMap
-            stdin_stmt = natural_language_map_translation(stdin_stmt, language)
-
-            # Perform translation based of translation stack.
-            assert visit_yaml_object(stdin_stmt, "str")
-            testcase["stdin"] = format_string(stdin_stmt, flat_stack, env)
-
-        # Translate NaturalLanguageMap
-        arguments = testcase.get("arguments", [])
-        arguments = natural_language_map_translation(arguments, language)
-
-        # Perform translation based of translation stack.
-        assert visit_yaml_object(arguments, "list")
-        testcase["arguments"] = parse_value(arguments, flat_stack, env)
-
-    if (stdout := testcase.get("stdout")) is not None:
-        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack, env)
-
-    if (file := testcase.get("file")) is not None:
-        # Translate NaturalLanguageMap
-        file = natural_language_map_translation(file, language)
-
-        assert visit_yaml_object(file, "dict")
-        file["content"] = format_string(str(file["content"]), flat_stack, env)
-        file["location"] = format_string(str(file["location"]), flat_stack, env)
-
-        testcase["file"] = file
-
-    if (stderr := testcase.get("stderr")) is not None:
-        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack, env)
-
-    if (exception := testcase.get("exception")) is not None:
-        testcase["exception"] = translate_io(
-            exception, "message", language, flat_stack, env
-        )
-
-    if (result := testcase.get("return")) is not None:
-        if visit_yaml_object(result, "ReturnOracle"):
-            arguments = result.get("arguments", [])
-            arguments = natural_language_map_translation(arguments, language)
+def wrap_in_braces(value):
+    return f"{{{value}}}"
 
-            # Perform translation based of translation stack.
-            result["arguments"] = parse_value(arguments, flat_stack, env)
 
-            value = result.get("value")
-            value = natural_language_map_translation(value, language)
+def create_enviroment() -> Environment:
+    enviroment = Environment()
+    enviroment.filters["braces"] = wrap_in_braces
+    return enviroment
 
-            result["value"] = parse_value(value, flat_stack, env)
-            testcase["return"] = result
 
-        elif visit_yaml_object(result, "NaturalLanguageMap"):
-            assert language in result
-            testcase["return"] = parse_value(result[language], flat_stack, env)
-        elif result is not None:
-            testcase["return"] = parse_value(result, flat_stack, env)
+def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
+    file_name = yaml_path.name
+    split_name = file_name.split(".")
+    path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
+    with open(path_to_new_yaml, "w") as yaml_file:
+        yaml_file.write(yaml_string)
 
-    if (description := testcase.get("description")) is not None:
-        description = natural_language_map_translation(description, language)
 
-        if visit_yaml_object(description, "str"):
-            testcase["description"] = format_string(description, flat_stack, env)
-        else:
-            assert visit_yaml_object(description, "dict")
-            dd = description["description"]
-            dd = natural_language_map_translation(dd, language)
+def convert_to_yaml(yaml_object: YamlObject) -> str:
+    def oracle_representer(dumper, data):
+        return dumper.represent_mapping("!oracle", data)
 
-            assert visit_yaml_object(dd, "str")
-            description["description"] = format_string(dd, flat_stack, env)
+    def expression_representer(dumper, data):
+        return dumper.represent_scalar("!expression", data)
 
-    testcase = translate_input_files(testcase, language, flat_stack, env)
+    def programming_language_map_representer(dumper, data):
+        return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data))
 
-    return testcase
+    # def represent_str(dumper, data):
+    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
 
+    # Register the representer for the ReturnOracle object
+    yaml.add_representer(ReturnOracle, oracle_representer)
+    yaml.add_representer(ExpressionString, expression_representer)
+    yaml.add_representer(ProgrammingLanguageMap, programming_language_map_representer)
+    # yaml.add_representer(str, represent_str)
+    return yaml.dump(yaml_object, sort_keys=False)
 
-def translate_testcases(
-    testcases: list, language: str, translation_stack: list, env: Environment
-) -> list:
-    result = []
-    for testcase in testcases:
-        assert visit_yaml_object(testcase, "dict")
-        result.append(translate_testcase(testcase, language, translation_stack, env))
 
+def parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any:
+    if isinstance(node, yaml.MappingNode):
+        result = loader.construct_mapping(node)
+    elif isinstance(node, yaml.SequenceNode):
+        result = loader.construct_sequence(node)
+    else:
+        assert isinstance(node, yaml.ScalarNode)
+        result = loader.construct_scalar(node)
     return result
 
 
-def translate_contexts(
-    contexts: list, language: str, translation_stack: list, env: Environment
-) -> list:
-    result = []
-    for context in contexts:
-        assert visit_yaml_object(context, "dict")
-
-        # Add translation to stack
-        if "translations" in context:
-            translation_stack.append(context["translations"])
-
-        key_to_set = "script" if "script" in context else "testcases"
-        raw_testcases = context.get(key_to_set)
-        assert visit_yaml_object(raw_testcases, "list")
-        context[key_to_set] = translate_testcases(
-            raw_testcases, language, translation_stack, env
-        )
-
-        flat_stack = flatten_stack(translation_stack, language)
-        context = translate_input_files(context, language, flat_stack, env)
-        result.append(context)
+def translate_map(value: YamlObject, language: str):
+    if isinstance(value, dict):
+        assert language in value
+        value = value[language]
+    return value
 
-        # Pop translation from stack
-        if "translations" in context:
-            translation_stack.pop()
-            context.pop("translations")
 
+def translate_translations_map(trans_map: dict, language: str) -> dict:
+    return {k: translate_map(v, language) for k, v in trans_map.items()}
+
+
+class State:
+    def __init__(
+        self, children: int, translations_stack: list, nat_language_indicator: list
+    ):
+        self.nat_language_indicator = nat_language_indicator
+        self.translations_stack = translations_stack
+        self.children = children
+        self.total_children = 0
+        for i in range(self.children):
+            if i < len(self.nat_language_indicator):
+                self.total_children += self.nat_language_indicator[i]
+            else:
+                self.total_children += 1
+
+    def is_finished(self) -> bool:
+        return self.total_children == 0
+
+
+class StateLoader(yaml.SafeLoader):
+    def __init__(self, stream):
+        super().__init__(stream)
+        self.level_state = 0
+        self.lang = ""
+        self.state_queue: deque[State] = deque()
+        start_state = State(1, [], [])
+        self.state_queue.append(start_state)
+        self.nat_language_indicator = []
+        self.env = create_enviroment()
+
+        self.tab_count = 0
+        self.tab_has_context = []
+        self.context_count = 0
+        self.tab_translations = {}
+        self.context_translations = {}
+        self.context_to_tab = {}
+
+    @staticmethod
+    def count_children(dictionary: dict):
+        return sum(1 for v in dictionary.values() if isinstance(v, list) and not v)
+
+    def set_language(self, lang: str):
+        self.lang = lang
+
+    def add_nat_language_indication(self, children: int):
+        self.nat_language_indicator.append(children)
+
+    def construct_mapping(self, node: yaml.Node, deep=False):
+        result = super().construct_mapping(node, deep)
+        if "tabs" in result or "units" in result:
+            assert (
+                self.level_state == 0
+            ), "Tabs can't be redefined or can't be defined when a tab is already defined."
+            self.level_state = 1
+
+        elif "tab" in result or "unit" in result:
+            assert (
+                self.level_state < 2
+            ), "Can't define a tab when a context or testcases are already defined."
+            self.level_state = 1
+
+        elif "testcases" in result or "scripts" in result:
+
+            if self.level_state == 1:
+                self.level_state = 2
+            assert (
+                self.level_state == 2
+            ), "Can't define a context when when a tab isn't defined yet."
+
+        children = self.count_children(result)
+        translation_stack = self.state_queue[0].translations_stack
+        if "translations" in result:
+            translation_stack.append(
+                translate_translations_map(result.pop("translations"), self.lang)
+            )
+            trans_map = flatten_stack(translation_stack)
+            result = parse_value(result, trans_map, self.env)
+
+        if children > 0:
+            new_state = State(children, translation_stack, self.nat_language_indicator)
+            self.state_queue.append(new_state)
+            self.nat_language_indicator = []
+
+        print(f"dict {result}")
+        return result
+
+    def construct_sequence(self, node: yaml.Node, deep=False):
+        result = super().construct_sequence(node, deep)
+
+        translation_stack = self.state_queue[0].translations_stack
+        trans_map = flatten_stack(translation_stack)
+        result = parse_value(result, trans_map, self.env)
+
+        self.state_queue[0].total_children -= 1
+        if self.state_queue[0].is_finished():
+            self.state_queue.popleft()
+
+        print(f"list {result}")
+        return result
+
+
+def natural_language_map(loader: StateLoader, node: yaml.Node) -> Any:
+    result = parse_yaml_value(loader, node)
+    print("nat_trans", result)
+    assert isinstance(
+        result, dict
+    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
+
+    children = loader.count_children(result)
+    loader.add_nat_language_indication(children)
+
+    return result[loader.lang]
+
+
+def dict_trans(loader: StateLoader, node: yaml.Node):
+    result = parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return result
 
 
-def translate_tab(
-    tab: YamlDict, language: str, translation_stack: list, env: Environment
-) -> YamlDict:
-    key_to_set = "unit" if "unit" in tab else "tab"
-    name = tab.get(key_to_set)
-    name = natural_language_map_translation(name, language)
-
-    assert visit_yaml_object(name, "str")
-    flat_stack = flatten_stack(translation_stack, language)
-    tab[key_to_set] = format_string(name, flat_stack, env)
+def seq_trans(loader: StateLoader, node: yaml.Node):
+    result = parse_yaml_value(loader, node)
+    assert isinstance(
+        result, list
+    ), f"A natural language map must be a list, got {result} which is a {type(result)}."
+    return result
 
-    tab = translate_input_files(tab, language, flat_stack, env)
 
-    # The tab can have testcases or contexts.
-    if "contexts" in tab:
-        assert visit_yaml_object(tab["contexts"], "list")
-        tab["contexts"] = translate_contexts(
-            tab["contexts"], language, translation_stack, env
-        )
-    elif "cases" in tab:
-        assert "unit" in tab
-        # We have testcases N.S. / contexts O.S.
-        assert visit_yaml_object(tab["cases"], "list")
-        tab["cases"] = translate_contexts(
-            tab["cases"], language, translation_stack, env
-        )
-    elif "testcases" in tab:
-        # We have scripts N.S. / testcases O.S.
-        assert "tab" in tab
-        assert visit_yaml_object(tab["testcases"], "list")
-        tab["testcases"] = translate_testcases(
-            tab["testcases"], language, translation_stack, env
-        )
-    else:
-        assert "scripts" in tab
-        assert visit_yaml_object(tab["scripts"], "list")
-        tab["scripts"] = translate_testcases(
-            tab["scripts"], language, translation_stack, env
+def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
+    """
+    Parse a string or stream to YAML.
+    """
+    try:
+        loader = StateLoader(yaml_stream)
+        loader.set_language(language)
+        for types in get_args(AllTypes):
+            for actual_type in types:
+                yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
+        yaml.add_constructor("!expression", _expression_string, loader)
+        yaml.add_constructor("!oracle", _return_oracle, loader)
+        yaml.add_constructor("!natural_language", natural_language_map, loader)
+        yaml.add_constructor("!programming_language", _programming_language_map, loader)
+        # Otherwise i won't have the full translations map on time
+        yaml.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_trans, loader
         )
-    return tab
-
 
-def translate_tabs(
-    dsl_list: list, language: str, env: Environment, translation_stack=None
-) -> list:
-    if translation_stack is None:
-        translation_stack = []
+        return loader.get_data()
+    except yaml.MarkedYAMLError as exc:
+        lines = yaml_stream.splitlines()
 
-    result = []
-    for tab in dsl_list:
-        assert visit_yaml_object(tab, "dict")
+        if exc.problem_mark is None:
+            # There is no additional information, so what can we do?
+            raise exc
 
-        if "translations" in tab:
-            translation_stack.append(tab["translations"])
+        sys.stderr.write(
+            textwrap.dedent(
+                f"""
+        YAML error while parsing test suite. This means there is a YAML syntax error.
 
-        result.append(translate_tab(tab, language, translation_stack, env))
-        if "translations" in tab:
-            translation_stack.pop()
-            tab.pop("translations")
+        The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}:
 
-    return result
+            {lines[exc.problem_mark.line]}
+            {" " * exc.problem_mark.column + "^"}
 
+        The error message was:
+            {exc.problem} {exc.context}
 
-def wrap_in_braces(value):
-    return f"{{{value}}}"
-
-
-def create_enviroment() -> Environment:
-    enviroment = Environment()
-    enviroment.filters["braces"] = wrap_in_braces
-    return enviroment
-
-
-def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
-
-    env = create_enviroment()
-
-    if visit_yaml_object(dsl_object, "lists"):
-        return translate_tabs(dsl_object, language, env)
-    else:
-        assert visit_yaml_object(dsl_object, "dict")
-        key_to_set = "units" if "units" in dsl_object else "tabs"
-        tab_list = dsl_object.get(key_to_set)
-        assert visit_yaml_object(tab_list, "list")
-        translation_stack = []
-        if "translations" in dsl_object:
-            translation_stack.append(dsl_object["translations"])
-            dsl_object.pop("translations")
-
-        flat_stack = flatten_stack(translation_stack, language)
-        dsl_object = translate_input_files(dsl_object, language, flat_stack, env)
-        dsl_object[key_to_set] = translate_tabs(
-            tab_list, language, env, translation_stack
+        The detailed exception is provided below.
+        You might also find help by validating your YAML file with a YAML validator.\n
+        """
+            )
         )
-        return dsl_object
+        raise exc
 
 
-def parse_yaml(yaml_path: Path) -> YamlObject:
+def parse_yaml(yaml_path: Path, language: str) -> YamlObject:
     with open(yaml_path, "r") as stream:
-        result = _parse_yaml(stream.read())
+        result = translate_yaml(stream.read(), language)
 
     return result
 
 
-def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
-    file_name = yaml_path.name
-    split_name = file_name.split(".")
-    path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
-    with open(path_to_new_yaml, "w") as yaml_file:
-        yaml_file.write(yaml_string)
-
-
-def convert_to_yaml(yaml_object: YamlObject) -> str:
-    def oracle_representer(dumper, data):
-        return dumper.represent_mapping("!oracle", data)
-
-    def expression_representer(dumper, data):
-        return dumper.represent_scalar("!expression", data)
-
-    # def represent_str(dumper, data):
-    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
-
-    # Register the representer for the ReturnOracle object
-    yaml.add_representer(ReturnOracle, oracle_representer)
-    yaml.add_representer(ExpressionString, expression_representer)
-    # yaml.add_representer(str, represent_str)
-    return yaml.dump(yaml_object, sort_keys=False)
-
-
 def run(path: Path, language: str):
-    new_yaml = parse_yaml(path)
-    validate_pre_dsl(new_yaml)
-    translated_dsl = translate_dsl(new_yaml, language)
-    yaml_string = convert_to_yaml(translated_dsl)
+    new_yaml = parse_yaml(path, language)
+    yaml_string = convert_to_yaml(new_yaml)
     _validate_dsl(_parse_yaml(yaml_string))
-
     generate_new_yaml(path, yaml_string, language)
 
 

From fe62210d0eec6dca18c74a61a8939bac8c99f6b3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 22 Feb 2025 19:02:35 +0100
Subject: [PATCH 040/111] fixed tests

---
 tested/dsl/translate_parser.py |  39 +----
 tested/nat_translation.py      | 260 +++++++++++++++++----------------
 tests/test_preprocess_dsl.py   |  42 +-----
 3 files changed, 147 insertions(+), 194 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 68a9529d..3f1b75f9 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -80,44 +80,20 @@ class TestedType:
     type: str | AllTypes
 
 
-# class ExpressionString(str):
-#     pass
-#
-#
-# class ReturnOracle(dict):
-#     pass
-#
-#
-# class NaturalLanguageMap(dict):
-#     pass
-#
-#
-# class ProgrammingLanguageMap(dict):
-#     pass
-
-
 class ExpressionString(str):
-    @staticmethod
-    def get_type_name() -> str:
-        return "ExpressionString"
+    pass
 
 
 class ReturnOracle(dict):
-    @staticmethod
-    def get_type_name() -> str:
-        return "ReturnOracle"
+    pass
 
 
 class NaturalLanguageMap(dict):
-    @staticmethod
-    def get_type_name() -> str:
-        return "NaturalLanguageMap"
+    pass
 
 
 class ProgrammingLanguageMap(dict):
-    @staticmethod
-    def get_type_name() -> str:
-        return "ProgrammingLanguageMap"
+    pass
 
 
 OptionDict = dict[str, int | bool]
@@ -136,13 +112,6 @@ def get_type_name() -> str:
 )
 
 
-# Function that calls translate if available
-def visit_yaml_object(obj: YamlObject, type_name: str) -> bool:
-    if hasattr(obj, "get_type_name"):
-        return obj.get_type_name() == type_name
-    return obj.__class__.__name__ == type_name
-
-
 def _convert_language_dictionary(
     original: dict[str, str]
 ) -> dict[SupportedLanguage, str]:
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index c29d27fe..755b6328 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,130 +1,29 @@
+import copy
 import sys
 import textwrap
 from collections import deque
 from pathlib import Path
-from typing import cast, Any
+from typing import Any, Hashable
 
 import yaml
 from jinja2 import Environment
 
 from tested.datatypes import AllTypes
 from tested.dsl.translate_parser import (
-    DslValidationError,
     ExpressionString,
-    NaturalLanguageMap,
     ProgrammingLanguageMap,
     ReturnOracle,
-    YamlDict,
     YamlObject,
-    _parse_yaml,
-    _validate_dsl,
-    _validate_testcase_combinations,
-    convert_validation_error_to_group,
-    load_schema_validator,
-    visit_yaml_object,
     _custom_type_constructors,
     _expression_string,
-    _return_oracle,
-    _natural_language_map,
+    _parse_yaml,
     _programming_language_map,
+    _return_oracle,
+    _validate_dsl,
 )
 from tested.utils import get_args
 
 
-def parse_value(
-    value: YamlObject, flattened_stack: dict, env: Environment
-) -> YamlObject:
-
-    # Will format the strings in different values.
-    if visit_yaml_object(value, "str"):
-        return format_string(value, flattened_stack, env)
-    elif visit_yaml_object(value, "dict"):
-        return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
-    elif visit_yaml_object(value, "list") and len(value) > 0:
-        return [parse_value(v, flattened_stack, env) for v in value]
-
-    return value
-
-
-def flatten_stack(translation_stack: list) -> dict:
-    # Will transform a list of translation maps into a dict that
-    # has all the keys defined over all the different translation map and will have
-    # the value of the newest definition. In this definition we also chose
-    # the translation of the provided language.
-    flattened = {}
-    for d in translation_stack:
-        flattened.update({k: v for k, v in d.items()})
-    return flattened
-
-
-def format_string(string: str, translations: dict, env: Environment) -> str:
-    template = env.from_string(string)
-    result = template.render(translations)
-
-    return result
-
-
-def wrap_in_braces(value):
-    return f"{{{value}}}"
-
-
-def create_enviroment() -> Environment:
-    enviroment = Environment()
-    enviroment.filters["braces"] = wrap_in_braces
-    return enviroment
-
-
-def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
-    file_name = yaml_path.name
-    split_name = file_name.split(".")
-    path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
-    with open(path_to_new_yaml, "w") as yaml_file:
-        yaml_file.write(yaml_string)
-
-
-def convert_to_yaml(yaml_object: YamlObject) -> str:
-    def oracle_representer(dumper, data):
-        return dumper.represent_mapping("!oracle", data)
-
-    def expression_representer(dumper, data):
-        return dumper.represent_scalar("!expression", data)
-
-    def programming_language_map_representer(dumper, data):
-        return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data))
-
-    # def represent_str(dumper, data):
-    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
-
-    # Register the representer for the ReturnOracle object
-    yaml.add_representer(ReturnOracle, oracle_representer)
-    yaml.add_representer(ExpressionString, expression_representer)
-    yaml.add_representer(ProgrammingLanguageMap, programming_language_map_representer)
-    # yaml.add_representer(str, represent_str)
-    return yaml.dump(yaml_object, sort_keys=False)
-
-
-def parse_yaml_value(loader: yaml.Loader, node: yaml.Node) -> Any:
-    if isinstance(node, yaml.MappingNode):
-        result = loader.construct_mapping(node)
-    elif isinstance(node, yaml.SequenceNode):
-        result = loader.construct_sequence(node)
-    else:
-        assert isinstance(node, yaml.ScalarNode)
-        result = loader.construct_scalar(node)
-    return result
-
-
-def translate_map(value: YamlObject, language: str):
-    if isinstance(value, dict):
-        assert language in value
-        value = value[language]
-    return value
-
-
-def translate_translations_map(trans_map: dict, language: str) -> dict:
-    return {k: translate_map(v, language) for k, v in trans_map.items()}
-
-
 class State:
     def __init__(
         self, children: int, translations_stack: list, nat_language_indicator: list
@@ -170,14 +69,21 @@ def set_language(self, lang: str):
 
     def add_nat_language_indication(self, children: int):
         self.nat_language_indicator.append(children)
+        if children > 0:
+            self.state_queue.pop()
 
-    def construct_mapping(self, node: yaml.Node, deep=False):
+    def construct_mapping(
+        self, node: yaml.MappingNode, deep=False
+    ) -> dict[Hashable, Any]:
         result = super().construct_mapping(node, deep)
+        translation_stack = copy.deepcopy(self.state_queue[0].translations_stack)
+
         if "tabs" in result or "units" in result:
             assert (
                 self.level_state == 0
             ), "Tabs can't be redefined or can't be defined when a tab is already defined."
             self.level_state = 1
+            self.state_queue.popleft()
 
         elif "tab" in result or "unit" in result:
             assert (
@@ -194,40 +100,148 @@ def construct_mapping(self, node: yaml.Node, deep=False):
             ), "Can't define a context when when a tab isn't defined yet."
 
         children = self.count_children(result)
-        translation_stack = self.state_queue[0].translations_stack
+
         if "translations" in result:
             translation_stack.append(
                 translate_translations_map(result.pop("translations"), self.lang)
             )
             trans_map = flatten_stack(translation_stack)
-            result = parse_value(result, trans_map, self.env)
+            result = parse_dict(result, trans_map, self.env)
 
+        print(f"dict {result}")
         if children > 0:
             new_state = State(children, translation_stack, self.nat_language_indicator)
             self.state_queue.append(new_state)
             self.nat_language_indicator = []
 
-        print(f"dict {result}")
         return result
 
-    def construct_sequence(self, node: yaml.Node, deep=False):
+    def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
         result = super().construct_sequence(node, deep)
-
         translation_stack = self.state_queue[0].translations_stack
         trans_map = flatten_stack(translation_stack)
-        result = parse_value(result, trans_map, self.env)
+        result = parse_list(result, trans_map, self.env)
 
         self.state_queue[0].total_children -= 1
         if self.state_queue[0].is_finished():
+            print(f"popping state {self.state_queue[0].translations_stack}")
             self.state_queue.popleft()
 
-        print(f"list {result}")
+        # print(f"list {result}")
         return result
 
 
+def parse_value(
+    value: YamlObject, flattened_stack: dict, env: Environment
+) -> YamlObject:
+
+    # Will format the strings in different values.
+    if isinstance(value, str):
+        return type(value)(format_string(value, flattened_stack, env))
+    elif isinstance(value, dict):
+        return type(value)(
+            {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
+        )
+    elif isinstance(value, list) and len(value) > 0:
+        return [parse_value(v, flattened_stack, env) for v in value]
+
+    return value
+
+
+def parse_list(value: list[Any], flattened_stack: dict, env: Environment) -> list[Any]:
+    if len(value) > 0:
+        return [parse_value(v, flattened_stack, env) for v in value]
+    return value
+
+
+def parse_dict(
+    value: dict[Hashable, Any], flattened_stack: dict, env: Environment
+) -> dict[Hashable, Any]:
+    return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
+
+
+def flatten_stack(translation_stack: list) -> dict:
+    # Will transform a list of translation maps into a dict that
+    # has all the keys defined over all the different translation map and will have
+    # the value of the newest definition. In this definition we also chose
+    # the translation of the provided language.
+    flattened = {}
+    for d in translation_stack:
+        flattened.update({k: v for k, v in d.items()})
+    return flattened
+
+
+def format_string(string: str, translations: dict, env: Environment) -> str:
+    template = env.from_string(string)
+    result = template.render(translations)
+
+    return result
+
+
+def wrap_in_braces(value):
+    return f"{{{value}}}"
+
+
+def create_enviroment() -> Environment:
+    enviroment = Environment()
+    enviroment.filters["braces"] = wrap_in_braces
+    return enviroment
+
+
+def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
+    file_name = yaml_path.name
+    split_name = file_name.split(".")
+    path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
+    with open(path_to_new_yaml, "w") as yaml_file:
+        yaml_file.write(yaml_string)
+
+
+def convert_to_yaml(yaml_object: YamlObject) -> str:
+    def oracle_representer(dumper, data):
+        return dumper.represent_mapping("!oracle", data)
+
+    def expression_representer(dumper, data):
+        return dumper.represent_scalar("!expression", data)
+
+    def programming_language_map_representer(dumper, data):
+        return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data))
+
+    # def represent_str(dumper, data):
+    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
+
+    # Register the representer for the ReturnOracle object
+    yaml.add_representer(ReturnOracle, oracle_representer)
+    yaml.add_representer(ExpressionString, expression_representer)
+    yaml.add_representer(ProgrammingLanguageMap, programming_language_map_representer)
+    # yaml.add_representer(str, represent_str)
+    return yaml.dump(yaml_object, sort_keys=False)
+
+
+def parse_yaml_value(loader: StateLoader, node: yaml.Node) -> Any:
+    if isinstance(node, yaml.MappingNode):
+        result = loader.construct_mapping(node)
+    elif isinstance(node, yaml.SequenceNode):
+        result = loader.construct_sequence(node)
+    else:
+        assert isinstance(node, yaml.ScalarNode)
+        result = loader.construct_scalar(node)
+    return result
+
+
+def translate_map(value: YamlObject, language: str):
+    if isinstance(value, dict):
+        assert language in value
+        value = value[language]
+    return value
+
+
+def translate_translations_map(trans_map: dict, language: str) -> dict:
+    return {k: translate_map(v, language) for k, v in trans_map.items()}
+
+
 def natural_language_map(loader: StateLoader, node: yaml.Node) -> Any:
     result = parse_yaml_value(loader, node)
-    print("nat_trans", result)
+    # print("nat_trans", result)
     assert isinstance(
         result, dict
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
@@ -263,14 +277,14 @@ def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
         loader.set_language(language)
         for types in get_args(AllTypes):
             for actual_type in types:
-                yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
-        yaml.add_constructor("!expression", _expression_string, loader)
-        yaml.add_constructor("!oracle", _return_oracle, loader)
-        yaml.add_constructor("!natural_language", natural_language_map, loader)
-        yaml.add_constructor("!programming_language", _programming_language_map, loader)
-        # Otherwise i won't have the full translations map on time
-        yaml.add_constructor(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_trans, loader
+                loader.add_constructor("!" + actual_type, _custom_type_constructors)
+        loader.add_constructor("!expression", _expression_string)
+        loader.add_constructor("!oracle", _return_oracle)
+        loader.add_constructor("!natural_language", natural_language_map)
+        loader.add_constructor("!programming_language", _programming_language_map)
+        # Otherwise you won't have the full translations map on time
+        loader.add_constructor(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_trans
         )
 
         return loader.get_data()
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 4c5fc1b4..829b0196 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,10 +1,9 @@
-from tested.dsl.translate_parser import _parse_yaml
+from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
     parse_value,
-    translate_dsl,
-    validate_pre_dsl,
+    translate_yaml,
 )
 
 
@@ -26,7 +25,7 @@ def test_natural_translate_unit_test():
     en: "select"
     nl: "selecteer"
 tabs:
-  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}_{{ negentien|default('{{ negentien }}') }}"
+  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}"
     translations:
       animal:
         en: "animal_tab"
@@ -105,7 +104,7 @@ def test_natural_translate_unit_test():
 """.strip()
     translated_yaml_str = """
 tabs:
-- tab: '{animal_tab}_{results}_{{ negentien }}'
+- tab: '{animal_tab}_{results}'
   contexts:
   - testcases:
     - statement: results_context = Trying(10)
@@ -149,10 +148,8 @@ def test_natural_translate_unit_test():
   - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
     return: 1
 """.strip()
-    parsed_yaml = _parse_yaml(yaml_str)
-    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_dsl = translate_yaml(yaml_str, "en")
     translated_yaml = convert_to_yaml(translated_dsl)
-    print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
 
@@ -249,10 +246,8 @@ def test_natural_translate_io_test():
   - expression: tests(11)
     return: 11
 """.strip()
-    parsed_yaml = _parse_yaml(yaml_str)
-    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_dsl = translate_yaml(yaml_str, "en")
     translated_yaml = convert_to_yaml(translated_dsl)
-    print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
 
@@ -271,28 +266,3 @@ def test_translate_parse():
     }
     parsed_result = parse_value(value, flattened_stack, env)
     assert parsed_result == expected_value
-
-
-def test_wrong_natural_translation_suite():
-    yaml_str = """
-tabs:
-- tab: animals
-  testcases:
-  - expression: tests(11)
-    return: 11
-  - expression:
-      javascript: animals_javascript(1 + 1)
-      typescript: animals_typescript(1 + 1)
-      java: Submission.animals_java(1 + 1)
-      python:
-        en: animals_python_en(1 + 1)
-        nl: animals_python_nl(1 + 1)
-    return: 2
-    """.strip()
-    parsed_yaml = _parse_yaml(yaml_str)
-    try:
-        validate_pre_dsl(parsed_yaml)
-    except ExceptionGroup:
-        print("As expected")
-    else:
-        assert False, "Expected ExceptionGroup, but no exception was raised"

From dec7057637a821bb81234113cf8f777bf3f56696 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 22 Feb 2025 19:07:28 +0100
Subject: [PATCH 041/111] removed some prints

---
 tested/nat_translation.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 755b6328..8db34d20 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -108,7 +108,6 @@ def construct_mapping(
             trans_map = flatten_stack(translation_stack)
             result = parse_dict(result, trans_map, self.env)
 
-        print(f"dict {result}")
         if children > 0:
             new_state = State(children, translation_stack, self.nat_language_indicator)
             self.state_queue.append(new_state)
@@ -124,10 +123,8 @@ def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
 
         self.state_queue[0].total_children -= 1
         if self.state_queue[0].is_finished():
-            print(f"popping state {self.state_queue[0].translations_stack}")
             self.state_queue.popleft()
 
-        # print(f"list {result}")
         return result
 
 
@@ -241,7 +238,6 @@ def translate_translations_map(trans_map: dict, language: str) -> dict:
 
 def natural_language_map(loader: StateLoader, node: yaml.Node) -> Any:
     result = parse_yaml_value(loader, node)
-    # print("nat_trans", result)
     assert isinstance(
         result, dict
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."

From 94ea109f81a1d72b812e2fa4c5d3e20fa3b8a775 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 13:45:37 +0100
Subject: [PATCH 042/111] cleaned up the code some more

---
 tested/dsl/translate_parser.py | 20 --------------
 tested/nat_translation.py      | 48 ++++++++++++----------------------
 2 files changed, 16 insertions(+), 52 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 3f1b75f9..68f54853 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -79,7 +79,6 @@ class TestedType:
     value: Any
     type: str | AllTypes
 
-
 class ExpressionString(str):
     pass
 
@@ -111,7 +110,6 @@ class ProgrammingLanguageMap(dict):
     | ProgrammingLanguageMap
 )
 
-
 def _convert_language_dictionary(
     original: dict[str, str]
 ) -> dict[SupportedLanguage, str]:
@@ -156,24 +154,6 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     return ReturnOracle(result)
 
 
-def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
-    result = _parse_yaml_value(loader, node)
-    assert isinstance(
-        result, dict
-    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
-    return NaturalLanguageMap(result)
-
-
-def _programming_language_map(
-    loader: yaml.Loader, node: yaml.Node
-) -> ProgrammingLanguageMap:
-    result = _parse_yaml_value(loader, node)
-    assert isinstance(
-        result, dict
-    ), f"A programming language map must be an object, got {result} which is a {type(result)}."
-    return ProgrammingLanguageMap(result)
-
-
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
     Parse a string or stream to YAML.
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 8db34d20..84638fb2 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -17,7 +17,6 @@
     _custom_type_constructors,
     _expression_string,
     _parse_yaml,
-    _programming_language_map,
     _return_oracle,
     _validate_dsl,
 )
@@ -214,21 +213,10 @@ def programming_language_map_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-def parse_yaml_value(loader: StateLoader, node: yaml.Node) -> Any:
-    if isinstance(node, yaml.MappingNode):
-        result = loader.construct_mapping(node)
-    elif isinstance(node, yaml.SequenceNode):
-        result = loader.construct_sequence(node)
-    else:
-        assert isinstance(node, yaml.ScalarNode)
-        result = loader.construct_scalar(node)
-    return result
-
-
 def translate_map(value: YamlObject, language: str):
-    if isinstance(value, dict):
-        assert language in value
-        value = value[language]
+    assert isinstance(value, dict)
+    assert language in value
+    value = value[language]
     return value
 
 
@@ -236,11 +224,8 @@ def translate_translations_map(trans_map: dict, language: str) -> dict:
     return {k: translate_map(v, language) for k, v in trans_map.items()}
 
 
-def natural_language_map(loader: StateLoader, node: yaml.Node) -> Any:
-    result = parse_yaml_value(loader, node)
-    assert isinstance(
-        result, dict
-    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
+def natural_language_map(loader: StateLoader, node: yaml.MappingNode) -> Any:
+    result = loader.construct_mapping(node)
 
     children = loader.count_children(result)
     loader.add_nat_language_indication(children)
@@ -248,20 +233,19 @@ def natural_language_map(loader: StateLoader, node: yaml.Node) -> Any:
     return result[loader.lang]
 
 
-def dict_trans(loader: StateLoader, node: yaml.Node):
-    result = parse_yaml_value(loader, node)
-    assert isinstance(
-        result, dict
-    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
-    return result
+def _programming_language_map(
+    loader: StateLoader, node: yaml.MappingNode
+) -> ProgrammingLanguageMap:
+    result = loader.construct_mapping(node)
+    return ProgrammingLanguageMap(result)
 
 
-def seq_trans(loader: StateLoader, node: yaml.Node):
-    result = parse_yaml_value(loader, node)
-    assert isinstance(
-        result, list
-    ), f"A natural language map must be a list, got {result} which is a {type(result)}."
-    return result
+def dict_trans(loader: StateLoader, node: yaml.MappingNode) -> dict[Hashable, Any]:
+    return loader.construct_mapping(node)
+
+
+def seq_trans(loader: StateLoader, node: yaml.SequenceNode):
+    return loader.construct_sequence(node)
 
 
 def translate_yaml(yaml_stream: str, language: str) -> YamlObject:

From c75d4a7470aeafd5ed8c701a1f8f3ce112852881 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 14:22:07 +0100
Subject: [PATCH 043/111] fixed linting issue and removed more redundant code.

---
 tested/dsl/translate_parser.py | 2 ++
 tested/nat_translation.py      | 8 +++-----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 68f54853..6d12419f 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -79,6 +79,7 @@ class TestedType:
     value: Any
     type: str | AllTypes
 
+
 class ExpressionString(str):
     pass
 
@@ -110,6 +111,7 @@ class ProgrammingLanguageMap(dict):
     | ProgrammingLanguageMap
 )
 
+
 def _convert_language_dictionary(
     original: dict[str, str]
 ) -> dict[SupportedLanguage, str]:
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 84638fb2..28b8d87c 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -214,7 +214,9 @@ def programming_language_map_representer(dumper, data):
 
 
 def translate_map(value: YamlObject, language: str):
-    assert isinstance(value, dict)
+    assert isinstance(
+        value, dict
+    ), "The translation map does not consist of dictionaries."
     assert language in value
     value = value[language]
     return value
@@ -244,10 +246,6 @@ def dict_trans(loader: StateLoader, node: yaml.MappingNode) -> dict[Hashable, An
     return loader.construct_mapping(node)
 
 
-def seq_trans(loader: StateLoader, node: yaml.SequenceNode):
-    return loader.construct_sequence(node)
-
-
 def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
     """
     Parse a string or stream to YAML.

From 7dac7d68798a42b6f539a5e31800880ef8732814 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 15:01:21 +0100
Subject: [PATCH 044/111] Removed some checks that are no longer used

---
 tested/dsl/translate_parser.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 6d12419f..47994e8c 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -205,14 +205,6 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
 
-def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
-    return isinstance(instance, NaturalLanguageMap)
-
-
-def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
-    return isinstance(instance, ProgrammingLanguageMap)
-
-
 def test(value: object) -> bool:
     if not isinstance(value, str):
         return False
@@ -231,12 +223,9 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         schema_object = json.load(schema_file)
 
     original_validator: Type[Validator] = validator_for(schema_object)
-    type_checker = (
-        original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
-        .redefine("expression", is_expression)
-        .redefine("natural_language", is_natural_language_map)
-        .redefine("programming_language", is_programming_language_map)
-    )
+    type_checker = original_validator.TYPE_CHECKER.redefine(
+        "oracle", is_oracle
+    ).redefine("expression", is_expression)
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)

From 86c64215acc6c115dc55ca7278682f04f830c399 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 15:19:14 +0100
Subject: [PATCH 045/111] Wat comments toegevoegd

---
 tested/nat_translation.py | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 28b8d87c..15a4a5e3 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -27,13 +27,13 @@ class State:
     def __init__(
         self, children: int, translations_stack: list, nat_language_indicator: list
     ):
-        self.nat_language_indicator = nat_language_indicator
+        self.nat_language_of_lists_indicator = nat_language_indicator
         self.translations_stack = translations_stack
         self.children = children
         self.total_children = 0
         for i in range(self.children):
-            if i < len(self.nat_language_indicator):
-                self.total_children += self.nat_language_indicator[i]
+            if i < len(self.nat_language_of_lists_indicator):
+                self.total_children += self.nat_language_of_lists_indicator[i]
             else:
                 self.total_children += 1
 
@@ -52,15 +52,10 @@ def __init__(self, stream):
         self.nat_language_indicator = []
         self.env = create_enviroment()
 
-        self.tab_count = 0
-        self.tab_has_context = []
-        self.context_count = 0
-        self.tab_translations = {}
-        self.context_translations = {}
-        self.context_to_tab = {}
-
     @staticmethod
     def count_children(dictionary: dict):
+        # The children of a dictionary will always be stored into a list that will
+        # be empty at the moment it is being parsed.
         return sum(1 for v in dictionary.values() if isinstance(v, list) and not v)
 
     def set_language(self, lang: str):
@@ -74,9 +69,12 @@ def add_nat_language_indication(self, children: int):
     def construct_mapping(
         self, node: yaml.MappingNode, deep=False
     ) -> dict[Hashable, Any]:
+        # This method will run for each map in a YamlObject.
         result = super().construct_mapping(node, deep)
         translation_stack = copy.deepcopy(self.state_queue[0].translations_stack)
 
+        # These checks are here to make sure that the order between tabs, a tab
+        # and testcases is preserved.
         if "tabs" in result or "units" in result:
             assert (
                 self.level_state == 0
@@ -104,6 +102,8 @@ def construct_mapping(
             translation_stack.append(
                 translate_translations_map(result.pop("translations"), self.lang)
             )
+            # A translation already happens here because when it should wait for the
+            # list that contains this dictionary, the state will not have this dictionary.
             trans_map = flatten_stack(translation_stack)
             result = parse_dict(result, trans_map, self.env)
 
@@ -116,6 +116,9 @@ def construct_mapping(
 
     def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
         result = super().construct_sequence(node, deep)
+        # After a lot of dictionaries have been parsed, the list will be
+        # parsed containing them. It is here that the formatting will take place.
+        # This way formatting will also happen as little as possible.
         translation_stack = self.state_queue[0].translations_stack
         trans_map = flatten_stack(translation_stack)
         result = parse_list(result, trans_map, self.env)
@@ -131,7 +134,6 @@ def parse_value(
     value: YamlObject, flattened_stack: dict, env: Environment
 ) -> YamlObject:
 
-    # Will format the strings in different values.
     if isinstance(value, str):
         return type(value)(format_string(value, flattened_stack, env))
     elif isinstance(value, dict):
@@ -202,14 +204,9 @@ def expression_representer(dumper, data):
     def programming_language_map_representer(dumper, data):
         return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data))
 
-    # def represent_str(dumper, data):
-    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
-
-    # Register the representer for the ReturnOracle object
     yaml.add_representer(ReturnOracle, oracle_representer)
     yaml.add_representer(ExpressionString, expression_representer)
     yaml.add_representer(ProgrammingLanguageMap, programming_language_map_representer)
-    # yaml.add_representer(str, represent_str)
     return yaml.dump(yaml_object, sort_keys=False)
 
 
@@ -260,7 +257,8 @@ def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
         loader.add_constructor("!oracle", _return_oracle)
         loader.add_constructor("!natural_language", natural_language_map)
         loader.add_constructor("!programming_language", _programming_language_map)
-        # Otherwise you won't have the full translations map on time
+        # This line is need because otherwise there won't
+        # be a full translations map on time.
         loader.add_constructor(
             yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_trans
         )

From e2d9d69e2636015a91e91b68e7a128f2e60180b9 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 16:15:27 +0100
Subject: [PATCH 046/111] gebruik van instanceof zoveel mogelijk vermeden

---
 tested/nat_translation.py    | 18 +++++-------------
 tests/test_preprocess_dsl.py | 19 ++++++++++++-------
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 15a4a5e3..e75eb406 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -102,10 +102,9 @@ def construct_mapping(
             translation_stack.append(
                 translate_translations_map(result.pop("translations"), self.lang)
             )
-            # A translation already happens here because when it should wait for the
-            # list that contains this dictionary, the state will not have this dictionary.
-            trans_map = flatten_stack(translation_stack)
-            result = parse_dict(result, trans_map, self.env)
+
+        trans_map = flatten_stack(translation_stack)
+        result = parse_dict(result, trans_map, self.env)
 
         if children > 0:
             new_state = State(children, translation_stack, self.nat_language_indicator)
@@ -116,9 +115,7 @@ def construct_mapping(
 
     def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
         result = super().construct_sequence(node, deep)
-        # After a lot of dictionaries have been parsed, the list will be
-        # parsed containing them. It is here that the formatting will take place.
-        # This way formatting will also happen as little as possible.
+
         translation_stack = self.state_queue[0].translations_stack
         trans_map = flatten_stack(translation_stack)
         result = parse_list(result, trans_map, self.env)
@@ -136,12 +133,6 @@ def parse_value(
 
     if isinstance(value, str):
         return type(value)(format_string(value, flattened_stack, env))
-    elif isinstance(value, dict):
-        return type(value)(
-            {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
-        )
-    elif isinstance(value, list) and len(value) > 0:
-        return [parse_value(v, flattened_stack, env) for v in value]
 
     return value
 
@@ -311,3 +302,4 @@ def run(path: Path, language: str):
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
 
     run(Path(sys.argv[1]), sys.argv[2])
+
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 829b0196..933f3213 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,9 +1,7 @@
-from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
-    parse_value,
-    translate_yaml,
+    translate_yaml, parse_dict, parse_list,
 )
 
 
@@ -253,16 +251,23 @@ def test_natural_translate_io_test():
 
 def test_translate_parse():
     env = create_enviroment()
-    flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
+    flattened_stack = {"human": "mens", "number": "getal"}
     value = {
-        "key1": ["value1_{{animal}}", "value1_{{human}}"],
+        "key1": "value1_{{human}}",
         "key2": "value2_{{number}}",
         "key3": 10,
     }
     expected_value = {
-        "key1": ["value1_dier", "value1_mens"],
+        "key1": "value1_mens",
         "key2": "value2_getal",
         "key3": 10,
     }
-    parsed_result = parse_value(value, flattened_stack, env)
+    parsed_result = parse_dict(value, flattened_stack, env)
+    print(parsed_result)
+    assert parsed_result == expected_value
+
+    value = ["value1_{{human}}", "value2_{{number}}",  10]
+    expected_value = ["value1_mens", "value2_getal",  10]
+    parsed_result = parse_list(value, flattened_stack, env)
+    print(parsed_result)
     assert parsed_result == expected_value

From 9a540a635e19929639c93548d20ff8a68f4e4d2f Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 17:00:13 +0100
Subject: [PATCH 047/111] Fixed linting en typing

---
 tested/nat_translation.py    |  1 -
 tests/test_preprocess_dsl.py | 19 +++++++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index e75eb406..6b4d6f26 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -302,4 +302,3 @@ def run(path: Path, language: str):
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
 
     run(Path(sys.argv[1]), sys.argv[2])
-
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 933f3213..cea6877d 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,7 +1,11 @@
+from typing import Any, Hashable, cast
+
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
-    translate_yaml, parse_dict, parse_list,
+    parse_dict,
+    parse_list,
+    translate_yaml,
 )
 
 
@@ -262,12 +266,15 @@ def test_translate_parse():
         "key2": "value2_getal",
         "key3": 10,
     }
-    parsed_result = parse_dict(value, flattened_stack, env)
-    print(parsed_result)
+    parsed_result = parse_dict(cast(dict[Hashable, Any], value), flattened_stack, env)
+    assert parsed_result == expected_value
+
+    value = ["value1_{{human}}", "value2_{{number}}", 10]
+    expected_value = ["value1_mens", "value2_getal", 10]
+    parsed_result = parse_list(value, flattened_stack, env)
     assert parsed_result == expected_value
 
-    value = ["value1_{{human}}", "value2_{{number}}",  10]
-    expected_value = ["value1_mens", "value2_getal",  10]
+    value = []
+    expected_value = []
     parsed_result = parse_list(value, flattened_stack, env)
-    print(parsed_result)
     assert parsed_result == expected_value

From c7d578772c8113adb6d042c93a13029281eb01c3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 23 Feb 2025 18:59:49 +0100
Subject: [PATCH 048/111] removed an unused field.

---
 tested/nat_translation.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 6b4d6f26..fc280895 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -29,9 +29,8 @@ def __init__(
     ):
         self.nat_language_of_lists_indicator = nat_language_indicator
         self.translations_stack = translations_stack
-        self.children = children
         self.total_children = 0
-        for i in range(self.children):
+        for i in range(children):
             if i < len(self.nat_language_of_lists_indicator):
                 self.total_children += self.nat_language_of_lists_indicator[i]
             else:
@@ -219,7 +218,6 @@ def natural_language_map(loader: StateLoader, node: yaml.MappingNode) -> Any:
 
     children = loader.count_children(result)
     loader.add_nat_language_indication(children)
-
     return result[loader.lang]
 
 

From 7681932fa6b7828189233379da4b6c48896f4413 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 24 Feb 2025 15:35:41 +0100
Subject: [PATCH 049/111] Fixed small bug

---
 tested/nat_translation.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index fc280895..75e283e1 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -81,12 +81,22 @@ def construct_mapping(
             self.level_state = 1
             self.state_queue.popleft()
 
+            if "translations" in result:
+                translation_stack.append(
+                    translate_translations_map(result.pop("translations"), self.lang)
+                )
+
         elif "tab" in result or "unit" in result:
             assert (
                 self.level_state < 2
             ), "Can't define a tab when a context or testcases are already defined."
             self.level_state = 1
 
+            if "translations" in result:
+                translation_stack.append(
+                    translate_translations_map(result.pop("translations"), self.lang)
+                )
+
         elif "testcases" in result or "scripts" in result:
 
             if self.level_state == 1:
@@ -95,12 +105,12 @@ def construct_mapping(
                 self.level_state == 2
             ), "Can't define a context when when a tab isn't defined yet."
 
-        children = self.count_children(result)
+            if "translations" in result:
+                translation_stack.append(
+                    translate_translations_map(result.pop("translations"), self.lang)
+                )
 
-        if "translations" in result:
-            translation_stack.append(
-                translate_translations_map(result.pop("translations"), self.lang)
-            )
+        children = self.count_children(result)
 
         trans_map = flatten_stack(translation_stack)
         result = parse_dict(result, trans_map, self.env)

From 820c36acf11be7cc98ab2354b0fd0185e906e25e Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 24 Feb 2025 16:01:11 +0100
Subject: [PATCH 050/111] Changed a name

---
 tested/nat_translation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 75e283e1..13e89b40 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -223,7 +223,7 @@ def translate_translations_map(trans_map: dict, language: str) -> dict:
     return {k: translate_map(v, language) for k, v in trans_map.items()}
 
 
-def natural_language_map(loader: StateLoader, node: yaml.MappingNode) -> Any:
+def _natural_language_map(loader: StateLoader, node: yaml.MappingNode) -> Any:
     result = loader.construct_mapping(node)
 
     children = loader.count_children(result)
@@ -254,7 +254,7 @@ def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
                 loader.add_constructor("!" + actual_type, _custom_type_constructors)
         loader.add_constructor("!expression", _expression_string)
         loader.add_constructor("!oracle", _return_oracle)
-        loader.add_constructor("!natural_language", natural_language_map)
+        loader.add_constructor("!natural_language", _natural_language_map)
         loader.add_constructor("!programming_language", _programming_language_map)
         # This line is need because otherwise there won't
         # be a full translations map on time.

From eecc7b17721988dcd08855d5347174165c0a88bb Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 27 Feb 2025 12:07:49 +0100
Subject: [PATCH 051/111] Merged all translations maps immediately

---
 tested/nat_translation.py | 47 +++++++++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 13e89b40..077bdc6b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -25,10 +25,10 @@
 
 class State:
     def __init__(
-        self, children: int, translations_stack: list, nat_language_indicator: list
+        self, children: int, translations_map: dict, nat_language_indicator: list
     ):
         self.nat_language_of_lists_indicator = nat_language_indicator
-        self.translations_stack = translations_stack
+        self.translations_map = translations_map
         self.total_children = 0
         for i in range(children):
             if i < len(self.nat_language_of_lists_indicator):
@@ -46,7 +46,7 @@ def __init__(self, stream):
         self.level_state = 0
         self.lang = ""
         self.state_queue: deque[State] = deque()
-        start_state = State(1, [], [])
+        start_state = State(1, {}, [])
         self.state_queue.append(start_state)
         self.nat_language_indicator = []
         self.env = create_enviroment()
@@ -70,7 +70,7 @@ def construct_mapping(
     ) -> dict[Hashable, Any]:
         # This method will run for each map in a YamlObject.
         result = super().construct_mapping(node, deep)
-        translation_stack = copy.deepcopy(self.state_queue[0].translations_stack)
+        new_translations_map = self.state_queue[0].translations_map
 
         # These checks are here to make sure that the order between tabs, a tab
         # and testcases is preserved.
@@ -82,8 +82,13 @@ def construct_mapping(
             self.state_queue.popleft()
 
             if "translations" in result:
-                translation_stack.append(
-                    translate_translations_map(result.pop("translations"), self.lang)
+                new_translations_map = flatten_stack(
+                    [
+                        new_translations_map,
+                        translate_translations_map(
+                            result.pop("translations"), self.lang
+                        ),
+                    ]
                 )
 
         elif "tab" in result or "unit" in result:
@@ -93,8 +98,13 @@ def construct_mapping(
             self.level_state = 1
 
             if "translations" in result:
-                translation_stack.append(
-                    translate_translations_map(result.pop("translations"), self.lang)
+                new_translations_map = flatten_stack(
+                    [
+                        new_translations_map,
+                        translate_translations_map(
+                            result.pop("translations"), self.lang
+                        ),
+                    ]
                 )
 
         elif "testcases" in result or "scripts" in result:
@@ -106,17 +116,22 @@ def construct_mapping(
             ), "Can't define a context when when a tab isn't defined yet."
 
             if "translations" in result:
-                translation_stack.append(
-                    translate_translations_map(result.pop("translations"), self.lang)
+                new_translations_map = flatten_stack(
+                    [
+                        new_translations_map,
+                        translate_translations_map(
+                            result.pop("translations"), self.lang
+                        ),
+                    ]
                 )
 
         children = self.count_children(result)
-
-        trans_map = flatten_stack(translation_stack)
-        result = parse_dict(result, trans_map, self.env)
+        result = parse_dict(result, new_translations_map, self.env)
 
         if children > 0:
-            new_state = State(children, translation_stack, self.nat_language_indicator)
+            new_state = State(
+                children, new_translations_map, self.nat_language_indicator
+            )
             self.state_queue.append(new_state)
             self.nat_language_indicator = []
 
@@ -125,9 +140,7 @@ def construct_mapping(
     def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
         result = super().construct_sequence(node, deep)
 
-        translation_stack = self.state_queue[0].translations_stack
-        trans_map = flatten_stack(translation_stack)
-        result = parse_list(result, trans_map, self.env)
+        result = parse_list(result, self.state_queue[0].translations_map, self.env)
 
         self.state_queue[0].total_children -= 1
         if self.state_queue[0].is_finished():

From 14e16db9575035b266ae85042585766dd9d2d13e Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 4 Mar 2025 16:23:03 +0100
Subject: [PATCH 052/111] re-added validator

---
 tested/dsl/schema-strict-nat-translation.json | 152 ++++++++++++----
 tested/dsl/translate_parser.py                |  71 +++++---
 tested/nat_translation.py                     | 163 ++++++++----------
 tests/test_preprocess_dsl.py                  | 105 ++++++++++-
 4 files changed, 336 insertions(+), 155 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 59cbc1c2..751f5499 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -80,7 +80,7 @@
             }
           ]
         },
-        "translation" : {
+        "translations" : {
           "type" : "object",
           "description": "Define translations in the global scope."
         },
@@ -94,18 +94,46 @@
       }
     },
     "_tabList" : {
-      "type" : "array",
-      "minItems" : 1,
-      "items" : {
-        "$ref" : "#/definitions/tab"
-      }
+      "oneOf" : [
+        {
+          "type" : "array",
+          "minItems" : 1,
+          "items" : {
+            "$ref" : "#/definitions/tab"
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "type" : "array",
+            "minItems" : 1,
+            "items" : {
+              "$ref" : "#/definitions/tab"
+            }
+          }
+        }
+      ]
     },
     "_unitList" : {
-      "type" : "array",
-      "minItems" : 1,
-      "items" : {
-        "$ref" : "#/definitions/unit"
-      }
+      "oneOf" : [
+        {
+          "type" : "array",
+          "minItems" : 1,
+          "items" : {
+            "$ref" : "#/definitions/unit"
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "type" : "array",
+            "minItems" : 1,
+            "items" : {
+              "$ref" : "#/definitions/unit"
+            }
+          }
+        }
+      ]
     },
     "tab" : {
       "type" : "object",
@@ -232,7 +260,7 @@
           ],
           "description" : "The name of this tab."
         },
-        "translation" : {
+        "translations" : {
           "type" : "object",
           "description": "Define translations in the unit scope."
         },
@@ -268,32 +296,88 @@
       ]
     },
     "_contextList" : {
-      "type" : "array",
-      "minItems" : 1,
-      "items" : {
-        "$ref" : "#/definitions/context"
-      }
+      "anyOf" : [
+        {
+          "type" : "array",
+          "minItems" : 1,
+          "items" : {
+            "$ref" : "#/definitions/context"
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "type" : "array",
+            "minItems" : 1,
+            "items" : {
+              "$ref" : "#/definitions/context"
+            }
+          }
+        }
+      ]
     },
     "_caseList" : {
-      "type" : "array",
-      "minItems" : 1,
-      "items" : {
-        "$ref" : "#/definitions/case"
-      }
+      "anyOf" : [
+        {
+          "type" : "array",
+          "minItems" : 1,
+          "items" : {
+            "$ref" : "#/definitions/case"
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "type" : "array",
+            "minItems" : 1,
+            "items" : {
+              "$ref" : "#/definitions/case"
+            }
+          }
+        }
+      ]
     },
     "_testcaseList" : {
-      "type" : "array",
-      "minItems" : 1,
-      "items" : {
-        "$ref" : "#/definitions/testcase"
-      }
+      "anyOf" : [
+        {
+          "type" : "array",
+          "minItems" : 1,
+          "items" : {
+            "$ref" : "#/definitions/testcase"
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "type" : "array",
+            "minItems" : 1,
+            "items" : {
+              "$ref" : "#/definitions/testcase"
+            }
+          }
+        }
+      ]
     },
     "_scriptList" : {
-      "type" : "array",
-      "minItems" : 1,
-      "items" : {
-        "$ref" : "#/definitions/script"
-      }
+      "anyOf" : [
+        {
+          "type" : "array",
+          "minItems" : 1,
+          "items" : {
+            "$ref" : "#/definitions/script"
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "type" : "array",
+            "minItems" : 1,
+            "items" : {
+              "$ref" : "#/definitions/script"
+            }
+          }
+        }
+      ]
     },
     "context" : {
       "type" : "object",
@@ -322,7 +406,7 @@
             }
           ]
         },
-        "translation" : {
+        "translations" : {
           "type" : "object",
           "description": "Define translations in the context scope."
         },
@@ -362,7 +446,7 @@
             }
           ]
         },
-        "translation" : {
+        "translations" : {
           "type" : "object",
           "description": "Define translations in the case scope."
         },
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 47994e8c..d5b47b2c 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -156,6 +156,34 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     return ReturnOracle(result)
 
 
+def raise_yaml_error(yaml_stream: str, exc: yaml.MarkedYAMLError):
+    lines = yaml_stream.splitlines()
+
+    if exc.problem_mark is None:
+        # There is no additional information, so what can we do?
+        raise exc
+
+    sys.stderr.write(
+        textwrap.dedent(
+            f"""
+            YAML error while parsing test suite. This means there is a YAML syntax error.
+
+            The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}:
+
+                {lines[exc.problem_mark.line]}
+                {" " * exc.problem_mark.column + "^"}
+
+            The error message was:
+                {exc.problem} {exc.context}
+
+            The detailed exception is provided below.
+            You might also find help by validating your YAML file with a YAML validator.\n
+            """
+        )
+    )
+    raise exc
+
+
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
     Parse a string or stream to YAML.
@@ -170,31 +198,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
     try:
         return yaml.load(yaml_stream, loader)
     except yaml.MarkedYAMLError as exc:
-        lines = yaml_stream.splitlines()
-
-        if exc.problem_mark is None:
-            # There is no additional information, so what can we do?
-            raise exc
-
-        sys.stderr.write(
-            textwrap.dedent(
-                f"""
-        YAML error while parsing test suite. This means there is a YAML syntax error.
-
-        The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}:
-            
-            {lines[exc.problem_mark.line]}
-            {" " * exc.problem_mark.column + "^"}
-        
-        The error message was:
-            {exc.problem} {exc.context}
-            
-        The detailed exception is provided below.
-        You might also find help by validating your YAML file with a YAML validator.\n
-        """
-            )
-        )
-        raise exc
+        raise_yaml_error(yaml_stream, exc)
 
 
 def is_oracle(_checker: TypeChecker, instance: Any) -> bool:
@@ -205,6 +209,14 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
 
+def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, NaturalLanguageMap)
+
+
+def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, ProgrammingLanguageMap)
+
+
 def test(value: object) -> bool:
     if not isinstance(value, str):
         return False
@@ -223,9 +235,12 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         schema_object = json.load(schema_file)
 
     original_validator: Type[Validator] = validator_for(schema_object)
-    type_checker = original_validator.TYPE_CHECKER.redefine(
-        "oracle", is_oracle
-    ).redefine("expression", is_expression)
+    type_checker = (
+        original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
+        .redefine("expression", is_expression)
+        .redefine("natural_language", is_natural_language_map)
+        .redefine("programming_language", is_programming_language_map)
+    )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 077bdc6b..36285920 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,28 +1,53 @@
-import copy
 import sys
-import textwrap
 from collections import deque
 from pathlib import Path
-from typing import Any, Hashable
+from typing import Any, Hashable, cast
 
 import yaml
 from jinja2 import Environment
 
 from tested.datatypes import AllTypes
 from tested.dsl.translate_parser import (
+    DslValidationError,
     ExpressionString,
+    NaturalLanguageMap,
     ProgrammingLanguageMap,
     ReturnOracle,
     YamlObject,
     _custom_type_constructors,
     _expression_string,
-    _parse_yaml,
     _return_oracle,
-    _validate_dsl,
+    convert_validation_error_to_group,
+    load_schema_validator,
+    raise_yaml_error,
 )
 from tested.utils import get_args
 
 
+def validate_pre_dsl(dsl_object: YamlObject):
+    """
+    Validate a DSl object.
+    :param dsl_object: The object to validate.
+    :return: True if valid, False otherwise.
+    """
+    _SCHEMA_VALIDATOR = load_schema_validator("schema-strict-nat-translation.json")
+    errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
+    if len(errors) == 1:
+        message = (
+            "Validating the DSL resulted in an error. "
+            "The most specific sub-exception is often the culprit. "
+        )
+        error = convert_validation_error_to_group(errors[0])
+        if isinstance(error, ExceptionGroup):
+            raise ExceptionGroup(message, error.exceptions)
+        else:
+            raise DslValidationError(message + str(error)) from error
+    elif len(errors) > 1:
+        the_errors = [convert_validation_error_to_group(e) for e in errors]
+        message = "Validating the DSL resulted in some errors."
+        raise ExceptionGroup(message, the_errors)
+
+
 class State:
     def __init__(
         self, children: int, translations_map: dict, nat_language_indicator: list
@@ -72,58 +97,16 @@ def construct_mapping(
         result = super().construct_mapping(node, deep)
         new_translations_map = self.state_queue[0].translations_map
 
-        # These checks are here to make sure that the order between tabs, a tab
-        # and testcases is preserved.
         if "tabs" in result or "units" in result:
-            assert (
-                self.level_state == 0
-            ), "Tabs can't be redefined or can't be defined when a tab is already defined."
-            self.level_state = 1
             self.state_queue.popleft()
 
-            if "translations" in result:
-                new_translations_map = flatten_stack(
-                    [
-                        new_translations_map,
-                        translate_translations_map(
-                            result.pop("translations"), self.lang
-                        ),
-                    ]
-                )
-
-        elif "tab" in result or "unit" in result:
-            assert (
-                self.level_state < 2
-            ), "Can't define a tab when a context or testcases are already defined."
-            self.level_state = 1
-
-            if "translations" in result:
-                new_translations_map = flatten_stack(
-                    [
-                        new_translations_map,
-                        translate_translations_map(
-                            result.pop("translations"), self.lang
-                        ),
-                    ]
-                )
-
-        elif "testcases" in result or "scripts" in result:
-
-            if self.level_state == 1:
-                self.level_state = 2
-            assert (
-                self.level_state == 2
-            ), "Can't define a context when when a tab isn't defined yet."
-
-            if "translations" in result:
-                new_translations_map = flatten_stack(
-                    [
-                        new_translations_map,
-                        translate_translations_map(
-                            result.pop("translations"), self.lang
-                        ),
-                    ]
-                )
+        if "translations" in result:
+            new_translations_map = flatten_stack(
+                [
+                    new_translations_map,
+                    translate_translations_map(result.pop("translations"), self.lang),
+                ]
+            )
 
         children = self.count_children(result)
         result = parse_dict(result, new_translations_map, self.env)
@@ -236,7 +219,9 @@ def translate_translations_map(trans_map: dict, language: str) -> dict:
     return {k: translate_map(v, language) for k, v in trans_map.items()}
 
 
-def _natural_language_map(loader: StateLoader, node: yaml.MappingNode) -> Any:
+def _natural_language_map_translation(
+    loader: StateLoader, node: yaml.MappingNode
+) -> Any:
     result = loader.construct_mapping(node)
 
     children = loader.count_children(result)
@@ -244,8 +229,15 @@ def _natural_language_map(loader: StateLoader, node: yaml.MappingNode) -> Any:
     return result[loader.lang]
 
 
-def _programming_language_map(
+def _natural_language_map(
     loader: StateLoader, node: yaml.MappingNode
+) -> NaturalLanguageMap:
+    result = loader.construct_mapping(node)
+    return NaturalLanguageMap(result)
+
+
+def _programming_language_map(
+    loader: yaml.Loader, node: yaml.MappingNode
 ) -> ProgrammingLanguageMap:
     result = loader.construct_mapping(node)
     return ProgrammingLanguageMap(result)
@@ -267,7 +259,7 @@ def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
                 loader.add_constructor("!" + actual_type, _custom_type_constructors)
         loader.add_constructor("!expression", _expression_string)
         loader.add_constructor("!oracle", _return_oracle)
-        loader.add_constructor("!natural_language", _natural_language_map)
+        loader.add_constructor("!natural_language", _natural_language_map_translation)
         loader.add_constructor("!programming_language", _programming_language_map)
         # This line is need because otherwise there won't
         # be a full translations map on time.
@@ -277,45 +269,42 @@ def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
 
         return loader.get_data()
     except yaml.MarkedYAMLError as exc:
-        lines = yaml_stream.splitlines()
+        raise_yaml_error(yaml_stream, exc)
 
-        if exc.problem_mark is None:
-            # There is no additional information, so what can we do?
-            raise exc
 
-        sys.stderr.write(
-            textwrap.dedent(
-                f"""
-        YAML error while parsing test suite. This means there is a YAML syntax error.
-
-        The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}:
-
-            {lines[exc.problem_mark.line]}
-            {" " * exc.problem_mark.column + "^"}
-
-        The error message was:
-            {exc.problem} {exc.context}
+def parse_yaml(yaml_stream: str) -> YamlObject:
+    """
+    Parse a string or stream to YAML.
+    """
+    loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.CSafeLoader)
+    for types in get_args(AllTypes):
+        for actual_type in types:
+            yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
+    yaml.add_constructor("!expression", _expression_string, loader)
+    yaml.add_constructor("!oracle", _return_oracle, loader)
+    yaml.add_constructor("!natural_language", _natural_language_map, loader)
+    yaml.add_constructor("!programming_language", _programming_language_map, loader)
 
-        The detailed exception is provided below.
-        You might also find help by validating your YAML file with a YAML validator.\n
-        """
-            )
-        )
-        raise exc
+    try:
+        return yaml.load(yaml_stream, loader)
+    except yaml.MarkedYAMLError as exc:
+        raise_yaml_error(yaml_stream, exc)
 
 
-def parse_yaml(yaml_path: Path, language: str) -> YamlObject:
+def read_yaml(yaml_path: Path):
     with open(yaml_path, "r") as stream:
-        result = translate_yaml(stream.read(), language)
-
-    return result
+        return stream.read()
 
 
 def run(path: Path, language: str):
-    new_yaml = parse_yaml(path, language)
-    yaml_string = convert_to_yaml(new_yaml)
-    _validate_dsl(_parse_yaml(yaml_string))
-    generate_new_yaml(path, yaml_string, language)
+    yaml_stream = read_yaml(path)
+    yaml_object = parse_yaml(yaml_stream)
+    validate_pre_dsl(yaml_object)
+
+    translated_yaml_ob = translate_yaml(yaml_stream, language)
+    translated_yaml_string = convert_to_yaml(translated_yaml_ob)
+    # _validate_dsl(_parse_yaml(yaml_string))
+    generate_new_yaml(path, translated_yaml_string, language)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index cea6877d..4abaf288 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,18 +1,17 @@
 from typing import Any, Hashable, cast
 
+from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
     parse_dict,
     parse_list,
+    parse_yaml,
     translate_yaml,
+    validate_pre_dsl,
 )
 
-
-def test_natural_translate_unit_test():
-    # Everywhere where !natural_language is used, it is mandatory to do so.
-    # Everywhere else it isn't.
-    yaml_str = """
+test_unit_yaml_str = """
 translations:
   animal:
     en: "animals"
@@ -103,7 +102,57 @@ def test_natural_translate_unit_test():
     testcases:
       - expression: "{{select}}('a', {'a': 1, 'b': 2})"
         return: 1
+  - tab: "task"
+    contexts: !natural_language
+        en:
+          - testcases:
+            - statement: '{{result}} = Trying(10)'
+            - expression: 'count_words({{result}})'
+              return: 'The {{result}} is 10'
+            - expression: !expression "count"
+              return: 'count'
+        nl:
+          - testcases:
+            - statement: '{{result}} = Proberen(10)'
+            - expression: 'tel_woorden({{result}})'
+              return: 'Het {{result}} is 10'
+            - expression: !expression "tellen"
+              return: 'tellen'
+  - tab: "task2"
+    contexts:
+        - testcases: !natural_language
+            en:
+              - statement: '{{result}} = Trying(10)'
+              - expression: 'count_words({{result}})'
+                return: 'The {{result}} is 10'
+              - expression: !expression "count"
+                return: 'count'
+            nl:
+              - statement: '{{result}} = Proberen(10)'
+              - expression: 'tel_woorden({{result}})'
+                return: 'Het {{result}} is 10'
+              - expression: !expression "tellen"
+                return: 'tellen'
+  - tab: "task3"
+    testcases: !natural_language
+        en:
+          - statement: '{{result}} = Trying(10)'
+          - expression: 'count_words({{result}})'
+            return: 'The {{result}} is 10'
+          - expression: !expression "count"
+            return: 'count'
+        nl:
+          - statement: '{{result}} = Proberen(10)'
+          - expression: 'tel_woorden({{result}})'
+            return: 'Het {{result}} is 10'
+          - expression: !expression "tellen"
+            return: 'tellen'
 """.strip()
+
+
+def test_natural_translate_unit_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
     translated_yaml_str = """
 tabs:
 - tab: '{animal_tab}_{results}'
@@ -149,8 +198,31 @@ def test_natural_translate_unit_test():
   testcases:
   - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
     return: 1
+- tab: task
+  contexts:
+  - testcases:
+    - statement: results = Trying(10)
+    - expression: count_words(results)
+      return: The results is 10
+    - expression: !expression 'count'
+      return: count
+- tab: task2
+  contexts:
+  - testcases:
+    - statement: results = Trying(10)
+    - expression: count_words(results)
+      return: The results is 10
+    - expression: !expression 'count'
+      return: count
+- tab: task3
+  testcases:
+  - statement: results = Trying(10)
+  - expression: count_words(results)
+    return: The results is 10
+  - expression: !expression 'count'
+    return: count
 """.strip()
-    translated_dsl = translate_yaml(yaml_str, "en")
+    translated_dsl = translate_yaml(test_unit_yaml_str, "en")
     translated_yaml = convert_to_yaml(translated_dsl)
     assert translated_yaml.strip() == translated_yaml_str
 
@@ -214,6 +286,14 @@ def test_natural_translate_io_test():
             en: "tests(11)"
             nl: "testen(11)"
         return: 11
+  - unit: "test2"
+    scripts: !natural_language
+      en: 
+        - expression: "tests(11)"
+          return: 11
+      nl:
+        - expression: "testen(11)"
+          return: 11
 """.strip()
     translated_yaml_str = """
 units:
@@ -247,6 +327,10 @@ def test_natural_translate_io_test():
   scripts:
   - expression: tests(11)
     return: 11
+- unit: test2
+  scripts:
+  - expression: tests(11)
+    return: 11
 """.strip()
     translated_dsl = translate_yaml(yaml_str, "en")
     translated_yaml = convert_to_yaml(translated_dsl)
@@ -278,3 +362,12 @@ def test_translate_parse():
     expected_value = []
     parsed_result = parse_list(value, flattened_stack, env)
     assert parsed_result == expected_value
+
+
+def test_validation():
+    yaml_object = parse_yaml(test_unit_yaml_str)
+    validate_pre_dsl(yaml_object)
+
+    translated_yaml_ob = translate_yaml(test_unit_yaml_str, "en")
+    translated_yaml_string = convert_to_yaml(translated_yaml_ob)
+    _validate_dsl(_parse_yaml(translated_yaml_string))

From 885aa48b744e107455f23531b0e1e6573468d96c Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 4 Mar 2025 16:54:38 +0100
Subject: [PATCH 053/111] Fixed typing issues

---
 tested/nat_translation.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 36285920..563e01b4 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,6 +16,7 @@
     YamlObject,
     _custom_type_constructors,
     _expression_string,
+    _parse_yaml_value,
     _return_oracle,
     convert_validation_error_to_group,
     load_schema_validator,
@@ -229,17 +230,21 @@ def _natural_language_map_translation(
     return result[loader.lang]
 
 
-def _natural_language_map(
-    loader: StateLoader, node: yaml.MappingNode
-) -> NaturalLanguageMap:
-    result = loader.construct_mapping(node)
+def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
 
 def _programming_language_map(
-    loader: yaml.Loader, node: yaml.MappingNode
+    loader: yaml.Loader, node: yaml.Node
 ) -> ProgrammingLanguageMap:
-    result = loader.construct_mapping(node)
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A programming language map must be an object, got {result} which is a {type(result)}."
     return ProgrammingLanguageMap(result)
 
 

From d06a55c2f821e2e8e66ece5b47cef4b55e518a24 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 4 Mar 2025 18:11:48 +0100
Subject: [PATCH 054/111] added test for error handling

---
 tested/nat_translation.py    | 10 +++-------
 tests/test_preprocess_dsl.py | 25 +++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 563e01b4..f851fefd 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -296,19 +296,15 @@ def parse_yaml(yaml_stream: str) -> YamlObject:
         raise_yaml_error(yaml_stream, exc)
 
 
-def read_yaml(yaml_path: Path):
-    with open(yaml_path, "r") as stream:
-        return stream.read()
-
-
 def run(path: Path, language: str):
-    yaml_stream = read_yaml(path)
+    with open(path, "r") as stream:
+        yaml_stream = stream.read()
+
     yaml_object = parse_yaml(yaml_stream)
     validate_pre_dsl(yaml_object)
 
     translated_yaml_ob = translate_yaml(yaml_stream, language)
     translated_yaml_string = convert_to_yaml(translated_yaml_ob)
-    # _validate_dsl(_parse_yaml(yaml_string))
     generate_new_yaml(path, translated_yaml_string, language)
 
 
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 4abaf288..8230c0b5 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -371,3 +371,28 @@ def test_validation():
     translated_yaml_ob = translate_yaml(test_unit_yaml_str, "en")
     translated_yaml_string = convert_to_yaml(translated_yaml_ob)
     _validate_dsl(_parse_yaml(translated_yaml_string))
+
+
+def test_wrong_natural_translation_suite():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python:
+        en: animals_python_en(1 + 1)
+        nl: animals_python_nl(1 + 1)
+    return: 2
+    """.strip()
+    parsed_yaml = parse_yaml(yaml_str)
+    try:
+        validate_pre_dsl(parsed_yaml)
+    except ExceptionGroup:
+        print("As expected")
+    else:
+        assert False, "Expected ExceptionGroup, but no exception was raised"

From b7c8c34d1aeb9a3677d837efa09945e3b571fe3a Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 4 Mar 2025 18:27:03 +0100
Subject: [PATCH 055/111] added an extra test for syntax errors

---
 tests/test_preprocess_dsl.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 8230c0b5..6ae4fc80 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,5 +1,5 @@
 from typing import Any, Hashable, cast
-
+import yaml
 from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
 from tested.nat_translation import (
     convert_to_yaml,
@@ -395,4 +395,28 @@ def test_wrong_natural_translation_suite():
     except ExceptionGroup:
         print("As expected")
     else:
-        assert False, "Expected ExceptionGroup, but no exception was raised"
+        assert False, "Expected ExceptionGroup error"
+
+def test_yaml_with_syntax_error():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression: !programming_language
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python: !nat_language
+        en: animals_python_en(1 + 1)
+        nl: animals_python_nl(1 + 1)
+    return: 2
+    """.strip()
+
+    try:
+        parse_yaml(yaml_str)
+    except yaml.MarkedYAMLError:
+        print("As expected")
+    else:
+        assert False, "Expected yaml.MarkedYAMLError error"

From 470e28c0ae52e4c477b2595e011f6454e3858e85 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 4 Mar 2025 18:27:51 +0100
Subject: [PATCH 056/111] fix linting

---
 tests/test_preprocess_dsl.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 6ae4fc80..6637489a 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,5 +1,7 @@
 from typing import Any, Hashable, cast
+
 import yaml
+
 from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
 from tested.nat_translation import (
     convert_to_yaml,
@@ -397,6 +399,7 @@ def test_wrong_natural_translation_suite():
     else:
         assert False, "Expected ExceptionGroup error"
 
+
 def test_yaml_with_syntax_error():
     yaml_str = """
 tabs:

From 94a766f4230cbf8ee5bef634b367628c4b727b45 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 7 Mar 2025 11:59:53 +0100
Subject: [PATCH 057/111] Added immediate link from preprocessor to tested.

---
 tested/__main__.py             | 10 ++++++++-
 tested/dsl/translate_parser.py |  4 ++--
 tested/main.py                 | 38 ++++++++++++++++++++--------------
 tested/nat_translation.py      | 30 ++++++++++++++++++++-------
 4 files changed, 57 insertions(+), 25 deletions(-)

diff --git a/tested/__main__.py b/tested/__main__.py
index 2f0c382d..f4fde274 100644
--- a/tested/__main__.py
+++ b/tested/__main__.py
@@ -30,6 +30,14 @@
     help="Include verbose logs. It is recommended to also use -o in this case.",
     action="store_true",
 )
+
+parser.add_argument(
+    "-t",
+    "--translate",
+    type=str,
+    help="Specifies the language to translate translate the dsl to.",
+    default='-'
+)
 parser = parser.parse_args()
 
 if parser.verbose:
@@ -42,4 +50,4 @@
 
 configuration = read_config(parser.config)
 with smart_close(parser.output) as out:
-    run(configuration, out)
+    run(configuration, out, parser.translate)
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index d5b47b2c..5e51018e 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -753,7 +753,7 @@ def _convert_dsl_list(
     return objects
 
 
-def _convert_dsl(dsl_object: YamlObject) -> Suite:
+def convert_dsl(dsl_object: YamlObject) -> Suite:
     """
     Translate a DSL test suite into a full test suite.
 
@@ -795,7 +795,7 @@ def parse_dsl(dsl_string: str) -> Suite:
     """
     dsl_object = _parse_yaml(dsl_string)
     _validate_dsl(dsl_object)
-    return _convert_dsl(dsl_object)
+    return convert_dsl(dsl_object)
 
 
 def translate_to_test_suite(dsl_string: str) -> str:
diff --git a/tested/main.py b/tested/main.py
index b4549028..081ec8d7 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -3,36 +3,44 @@
 """
 
 import os
+from pathlib import Path
 from typing import IO
 
 from tested.configs import DodonaConfig, create_bundle
 from tested.dsl import parse_dsl
+from tested.dsl.translate_parser import convert_dsl
+from tested.nat_translation import run_translation
 from tested.testsuite import parse_test_suite
 
 
-def run(config: DodonaConfig, judge_output: IO):
+def run(config: DodonaConfig, judge_output: IO, language: str):
     """
     Run the TESTed judge.
 
     :param config: The configuration, as received from Dodona.
     :param judge_output: Where the judge output will be written to.
+    :param language: The language to use to translate the test-suite.
     """
-    try:
-        with open(f"{config.resources}/{config.test_suite}", "r") as t:
-            textual_suite = t.read()
-    except FileNotFoundError as e:
-        print("The test suite was not found. Check your exercise's config.json file.")
-        print(
-            "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
-        )
-        raise e
+    if language == "-":
+        try:
+            with open(f"{config.resources}/{config.test_suite}", "r") as t:
+                textual_suite = t.read()
+        except FileNotFoundError as e:
+            print("The test suite was not found. Check your exercise's config.json file.")
+            print(
+                "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
+            )
+            raise e
 
-    _, ext = os.path.splitext(config.test_suite)
-    is_yaml = ext.lower() in (".yaml", ".yml")
-    if is_yaml:
-        suite = parse_dsl(textual_suite)
+        _, ext = os.path.splitext(config.test_suite)
+        is_yaml = ext.lower() in (".yaml", ".yml")
+        if is_yaml:
+            suite = parse_dsl(textual_suite)
+        else:
+            suite = parse_test_suite(textual_suite)
     else:
-        suite = parse_test_suite(textual_suite)
+        translated_yaml_ob = run_translation(Path(f"{config.resources}/{config.test_suite}"), language=language, to_file=False)
+        suite = convert_dsl(translated_yaml_ob)
     pack = create_bundle(config, judge_output, suite)
     from .judge import judge
 
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index f851fefd..cb5ee65d 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,6 @@
+import os
 import sys
+from argparse import ArgumentParser
 from collections import deque
 from pathlib import Path
 from typing import Any, Hashable, cast
@@ -20,7 +22,7 @@
     _return_oracle,
     convert_validation_error_to_group,
     load_schema_validator,
-    raise_yaml_error,
+    raise_yaml_error, _validate_dsl, _parse_yaml,
 )
 from tested.utils import get_args
 
@@ -296,20 +298,34 @@ def parse_yaml(yaml_stream: str) -> YamlObject:
         raise_yaml_error(yaml_stream, exc)
 
 
-def run(path: Path, language: str):
-    with open(path, "r") as stream:
-        yaml_stream = stream.read()
-
+def run_translation(path: Path, language: str, to_file: bool =True) -> YamlObject:
+    try:
+        with open(path, "r") as stream:
+            yaml_stream = stream.read()
+    except FileNotFoundError as e:
+        print("The test suite was not found. Check your exercise's config.json file.")
+        print(
+            "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
+        )
+        raise e
+    _, ext = os.path.splitext(path)
+    assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}."
     yaml_object = parse_yaml(yaml_stream)
     validate_pre_dsl(yaml_object)
 
     translated_yaml_ob = translate_yaml(yaml_stream, language)
     translated_yaml_string = convert_to_yaml(translated_yaml_ob)
-    generate_new_yaml(path, translated_yaml_string, language)
+    _validate_dsl(_parse_yaml(translated_yaml_string))
+    if to_file:
+        generate_new_yaml(path, translated_yaml_string, language)
+
+    return translated_yaml_ob
+
+
 
 
 if __name__ == "__main__":
     n = len(sys.argv)
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
 
-    run(Path(sys.argv[1]), sys.argv[2])
+    run_translation(Path(sys.argv[1]), sys.argv[2])

From 462913447741e1967f4d404133ca423ba95265db Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 7 Mar 2025 12:20:27 +0100
Subject: [PATCH 058/111] Fixed bug for a lot of tests

---
 tested/__main__.py        |  2 +-
 tested/main.py            | 12 +++++++++---
 tested/nat_translation.py |  8 ++++----
 3 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/tested/__main__.py b/tested/__main__.py
index f4fde274..72f9f085 100644
--- a/tested/__main__.py
+++ b/tested/__main__.py
@@ -36,7 +36,7 @@
     "--translate",
     type=str,
     help="Specifies the language to translate translate the dsl to.",
-    default='-'
+    default="-",
 )
 parser = parser.parse_args()
 
diff --git a/tested/main.py b/tested/main.py
index 081ec8d7..c7052208 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -13,7 +13,7 @@
 from tested.testsuite import parse_test_suite
 
 
-def run(config: DodonaConfig, judge_output: IO, language: str):
+def run(config: DodonaConfig, judge_output: IO, language: str = "-"):
     """
     Run the TESTed judge.
 
@@ -26,7 +26,9 @@ def run(config: DodonaConfig, judge_output: IO, language: str):
             with open(f"{config.resources}/{config.test_suite}", "r") as t:
                 textual_suite = t.read()
         except FileNotFoundError as e:
-            print("The test suite was not found. Check your exercise's config.json file.")
+            print(
+                "The test suite was not found. Check your exercise's config.json file."
+            )
             print(
                 "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
             )
@@ -39,7 +41,11 @@ def run(config: DodonaConfig, judge_output: IO, language: str):
         else:
             suite = parse_test_suite(textual_suite)
     else:
-        translated_yaml_ob = run_translation(Path(f"{config.resources}/{config.test_suite}"), language=language, to_file=False)
+        translated_yaml_ob = run_translation(
+            Path(f"{config.resources}/{config.test_suite}"),
+            language=language,
+            to_file=False,
+        )
         suite = convert_dsl(translated_yaml_ob)
     pack = create_bundle(config, judge_output, suite)
     from .judge import judge
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index cb5ee65d..7fee4b46 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -18,11 +18,13 @@
     YamlObject,
     _custom_type_constructors,
     _expression_string,
+    _parse_yaml,
     _parse_yaml_value,
     _return_oracle,
+    _validate_dsl,
     convert_validation_error_to_group,
     load_schema_validator,
-    raise_yaml_error, _validate_dsl, _parse_yaml,
+    raise_yaml_error,
 )
 from tested.utils import get_args
 
@@ -298,7 +300,7 @@ def parse_yaml(yaml_stream: str) -> YamlObject:
         raise_yaml_error(yaml_stream, exc)
 
 
-def run_translation(path: Path, language: str, to_file: bool =True) -> YamlObject:
+def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObject:
     try:
         with open(path, "r") as stream:
             yaml_stream = stream.read()
@@ -322,8 +324,6 @@ def run_translation(path: Path, language: str, to_file: bool =True) -> YamlObjec
     return translated_yaml_ob
 
 
-
-
 if __name__ == "__main__":
     n = len(sys.argv)
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."

From f9d311d4093d7b892f04002b2be33adaa78a5fa4 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 7 Mar 2025 12:21:29 +0100
Subject: [PATCH 059/111] Small cleanup

---
 tested/nat_translation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 7fee4b46..57af5f4b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,6 +1,5 @@
 import os
 import sys
-from argparse import ArgumentParser
 from collections import deque
 from pathlib import Path
 from typing import Any, Hashable, cast

From f7c65cb21b21e0642e16da52441a3d36af13e464 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 7 Mar 2025 13:35:28 +0100
Subject: [PATCH 060/111] made a few more tests

---
 tests/test_preprocess_dsl.py | 47 +++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 6637489a..a5db8fd7 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,7 +1,12 @@
+from pathlib import Path
 from typing import Any, Hashable, cast
+from unittest.mock import ANY
 
 import yaml
+import pytest
+from pytest_mock import MockerFixture
 
+import tested
 from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
 from tested.nat_translation import (
     convert_to_yaml,
@@ -10,7 +15,7 @@
     parse_list,
     parse_yaml,
     translate_yaml,
-    validate_pre_dsl,
+    validate_pre_dsl, run_translation,
 )
 
 test_unit_yaml_str = """
@@ -423,3 +428,43 @@ def test_yaml_with_syntax_error():
         print("As expected")
     else:
         assert False, "Expected yaml.MarkedYAMLError error"
+
+
+def test_run_is_correct(mocker: MockerFixture):
+    s = mocker.spy(tested.nat_translation, name="generate_new_yaml")  # type: ignore[reportAttributeAccessIssue]
+    mock_files = [
+        mocker.mock_open(read_data=content).return_value
+        for content in ["""
+tabs:
+- tab: task3
+  testcases:
+  - statement: !natural_language
+        nl: resultaten = Proberen(10)
+        en: results = Tries(10)
+  - expression: !natural_language
+        nl: tel_woorden(resultaten)
+        en: count_words(results)
+    return: !natural_language
+        nl: Het resultaat is 10
+        en: The result is 10"""]
+    ]
+    mock_files.append(mocker.mock_open(read_data="{}").return_value)
+    mock_opener = mocker.mock_open()
+    mock_opener.side_effect = mock_files
+    mocker.patch("builtins.open", mock_opener)
+
+    yaml_object = run_translation(Path("suite.yaml"), "en", False)
+
+    assert s.call_count == 0
+    assert yaml_object["tabs"][0]["testcases"][0] == {'statement': 'results = Tries(10)'}
+    assert yaml_object["tabs"][0]["testcases"][1] == {'expression': 'count_words(results)',
+                                                      'return': 'The result is 10'}
+
+def test_run_is_correct_when_no_file():
+
+    try:
+        run_translation(Path("suite.yaml"), "en", False)
+    except FileNotFoundError as e:
+        print("As expected")
+    else:
+        assert False, "Expected FileNotFoundError error"

From a53905344abadd3348f03bd5201fbc9911c4e993 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 7 Mar 2025 15:38:26 +0100
Subject: [PATCH 061/111] fixed linting issues

---
 tests/test_preprocess_dsl.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index a5db8fd7..7a72c334 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -2,8 +2,8 @@
 from typing import Any, Hashable, cast
 from unittest.mock import ANY
 
-import yaml
 import pytest
+import yaml
 from pytest_mock import MockerFixture
 
 import tested
@@ -14,8 +14,9 @@
     parse_dict,
     parse_list,
     parse_yaml,
+    run_translation,
     translate_yaml,
-    validate_pre_dsl, run_translation,
+    validate_pre_dsl,
 )
 
 test_unit_yaml_str = """
@@ -434,7 +435,8 @@ def test_run_is_correct(mocker: MockerFixture):
     s = mocker.spy(tested.nat_translation, name="generate_new_yaml")  # type: ignore[reportAttributeAccessIssue]
     mock_files = [
         mocker.mock_open(read_data=content).return_value
-        for content in ["""
+        for content in [
+            """
 tabs:
 - tab: task3
   testcases:
@@ -446,7 +448,8 @@ def test_run_is_correct(mocker: MockerFixture):
         en: count_words(results)
     return: !natural_language
         nl: Het resultaat is 10
-        en: The result is 10"""]
+        en: The result is 10"""
+        ]
     ]
     mock_files.append(mocker.mock_open(read_data="{}").return_value)
     mock_opener = mocker.mock_open()
@@ -456,9 +459,15 @@ def test_run_is_correct(mocker: MockerFixture):
     yaml_object = run_translation(Path("suite.yaml"), "en", False)
 
     assert s.call_count == 0
-    assert yaml_object["tabs"][0]["testcases"][0] == {'statement': 'results = Tries(10)'}
-    assert yaml_object["tabs"][0]["testcases"][1] == {'expression': 'count_words(results)',
-                                                      'return': 'The result is 10'}
+    assert isinstance(yaml_object, dict)
+    tabs = yaml_object["tabs"]
+    assert isinstance(tabs, list)
+    assert tabs[0]["testcases"][0] == {"statement": "results = Tries(10)"}
+    assert tabs[0]["testcases"][1] == {
+        "expression": "count_words(results)",
+        "return": "The result is 10",
+    }
+
 
 def test_run_is_correct_when_no_file():
 

From c72190c833604fbb74452fd765f3d325603d0629 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 18:25:29 +0100
Subject: [PATCH 062/111] fixed small issue regarding lists of tabs

---
 tested/nat_translation.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 57af5f4b..de51e62d 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -75,7 +75,6 @@ def __init__(self, stream):
         self.level_state = 0
         self.lang = ""
         self.state_queue: deque[State] = deque()
-        start_state = State(1, {}, [])
         self.state_queue.append(start_state)
         self.nat_language_indicator = []
         self.env = create_enviroment()
@@ -99,10 +98,10 @@ def construct_mapping(
     ) -> dict[Hashable, Any]:
         # This method will run for each map in a YamlObject.
         result = super().construct_mapping(node, deep)
-        new_translations_map = self.state_queue[0].translations_map
+        new_translations_map = {}
+        if len(self.state_queue) > 0:
+            new_translations_map = self.state_queue[0].translations_map
 
-        if "tabs" in result or "units" in result:
-            self.state_queue.popleft()
 
         if "translations" in result:
             new_translations_map = flatten_stack(

From 0959919b444d9ca1233f2dfb9129ce018d47de69 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 18:28:34 +0100
Subject: [PATCH 063/111] removed field to serves no purpose anymore

---
 tested/nat_translation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index de51e62d..a694ad39 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -72,7 +72,6 @@ def is_finished(self) -> bool:
 class StateLoader(yaml.SafeLoader):
     def __init__(self, stream):
         super().__init__(stream)
-        self.level_state = 0
         self.lang = ""
         self.state_queue: deque[State] = deque()
         self.state_queue.append(start_state)

From 8aee618e916c6c9ec08a5dd382802a77142e8385 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 18:34:17 +0100
Subject: [PATCH 064/111] fixed linting

---
 tested/nat_translation.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index a694ad39..78542b79 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -74,7 +74,6 @@ def __init__(self, stream):
         super().__init__(stream)
         self.lang = ""
         self.state_queue: deque[State] = deque()
-        self.state_queue.append(start_state)
         self.nat_language_indicator = []
         self.env = create_enviroment()
 
@@ -101,7 +100,6 @@ def construct_mapping(
         if len(self.state_queue) > 0:
             new_translations_map = self.state_queue[0].translations_map
 
-
         if "translations" in result:
             new_translations_map = flatten_stack(
                 [

From f6685354c0458337f5c785ed177505b702b75203 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 19:29:30 +0100
Subject: [PATCH 065/111] Found another edge case that wasn't covered

---
 tested/nat_translation.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 78542b79..3c208150 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -58,12 +58,10 @@ def __init__(
     ):
         self.nat_language_of_lists_indicator = nat_language_indicator
         self.translations_map = translations_map
-        self.total_children = 0
-        for i in range(children):
-            if i < len(self.nat_language_of_lists_indicator):
-                self.total_children += self.nat_language_of_lists_indicator[i]
-            else:
-                self.total_children += 1
+        self.total_children = sum(self.nat_language_of_lists_indicator)
+
+        if children > len(self.nat_language_of_lists_indicator):
+            self.total_children += children - len(self.nat_language_of_lists_indicator)
 
     def is_finished(self) -> bool:
         return self.total_children == 0

From b62bfc32ecb060493ede621a450afd232baf38b1 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 19:36:59 +0100
Subject: [PATCH 066/111] using pythonic code

---
 tested/nat_translation.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 3c208150..cd19ae04 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -94,9 +94,9 @@ def construct_mapping(
     ) -> dict[Hashable, Any]:
         # This method will run for each map in a YamlObject.
         result = super().construct_mapping(node, deep)
-        new_translations_map = {}
-        if len(self.state_queue) > 0:
-            new_translations_map = self.state_queue[0].translations_map
+        new_translations_map = (
+            self.state_queue[0].translations_map if self.state_queue else {}
+        )
 
         if "translations" in result:
             new_translations_map = flatten_stack(

From 9f4a84f56e168702fa73d2da1102934e43f55e7e Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 19:53:07 +0100
Subject: [PATCH 067/111] Fixed a small bug with the nat_lang_indicators

---
 tested/nat_translation.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index cd19ae04..b8f1f91f 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -85,9 +85,14 @@ def set_language(self, lang: str):
         self.lang = lang
 
     def add_nat_language_indication(self, children: int):
-        self.nat_language_indicator.append(children)
+
+        # construct_mapping runs before this, which will construct a false state
         if children > 0:
-            self.state_queue.pop()
+            false_state = self.state_queue.pop()
+            # restore nat_language_indicator because it shouldn't of been emptied.
+            self.nat_language_indicator = false_state.nat_language_of_lists_indicator
+
+        self.nat_language_indicator.append(children)
 
     def construct_mapping(
         self, node: yaml.MappingNode, deep=False

From 23d9b01da19c89444befde03739f5cde8e2cb394 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 21:21:38 +0100
Subject: [PATCH 068/111] Added another check

---
 tested/nat_translation.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b8f1f91f..6fa0504e 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -227,6 +227,8 @@ def _natural_language_map_translation(
 ) -> Any:
     result = loader.construct_mapping(node)
 
+    all_types = {type(v) for v in result.values()}
+    assert len(all_types) == 1, "A natural language map should have values with the same type."
     children = loader.count_children(result)
     loader.add_nat_language_indication(children)
     return result[loader.lang]

From e78c518d8803d24e0bd7db31d67d986202e4f33d Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 21:22:14 +0100
Subject: [PATCH 069/111] Fxed linting

---
 tested/nat_translation.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 6fa0504e..2f1f3244 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -228,7 +228,9 @@ def _natural_language_map_translation(
     result = loader.construct_mapping(node)
 
     all_types = {type(v) for v in result.values()}
-    assert len(all_types) == 1, "A natural language map should have values with the same type."
+    assert (
+        len(all_types) == 1
+    ), "A natural language map should have values with the same type."
     children = loader.count_children(result)
     loader.add_nat_language_indication(children)
     return result[loader.lang]

From 3ed7491dfd3a8ede98b873503f154a08be60d4c0 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 12 Mar 2025 21:28:42 +0100
Subject: [PATCH 070/111] forgot a tab

---
 tested/nat_translation.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 2f1f3244..7467dbea 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -91,8 +91,7 @@ def add_nat_language_indication(self, children: int):
             false_state = self.state_queue.pop()
             # restore nat_language_indicator because it shouldn't of been emptied.
             self.nat_language_indicator = false_state.nat_language_of_lists_indicator
-
-        self.nat_language_indicator.append(children)
+            self.nat_language_indicator.append(children)
 
     def construct_mapping(
         self, node: yaml.MappingNode, deep=False

From 2027194a4dbe402748da576b602ff914d603fe0c Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 14 Mar 2025 11:32:16 +0100
Subject: [PATCH 071/111] fixed problem when no translations are used

---
 tested/nat_translation.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 7467dbea..b3ab02d9 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -125,11 +125,16 @@ def construct_mapping(
     def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
         result = super().construct_sequence(node, deep)
 
-        result = parse_list(result, self.state_queue[0].translations_map, self.env)
+        translations_map = (
+            self.state_queue[0].translations_map if self.state_queue else {}
+        )
+
+        result = parse_list(result, translations_map, self.env)
 
-        self.state_queue[0].total_children -= 1
-        if self.state_queue[0].is_finished():
-            self.state_queue.popleft()
+        if len(self.state_queue) > 0:
+            self.state_queue[0].total_children -= 1
+            if self.state_queue[0].is_finished():
+                self.state_queue.popleft()
 
         return result
 
@@ -317,6 +322,7 @@ def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObje
     validate_pre_dsl(yaml_object)
 
     translated_yaml_ob = translate_yaml(yaml_stream, language)
+    print(f"result: {translated_yaml_ob}")
     translated_yaml_string = convert_to_yaml(translated_yaml_ob)
     _validate_dsl(_parse_yaml(translated_yaml_string))
     if to_file:

From 9055b90b1b0cc4336e30c56bb9f02f7c35b722c9 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 14 Mar 2025 12:07:55 +0100
Subject: [PATCH 072/111] kleine optimalisatie

---
 tested/nat_translation.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b3ab02d9..2e90ddc3 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -86,15 +86,11 @@ def set_language(self, lang: str):
 
     def add_nat_language_indication(self, children: int):
 
-        # construct_mapping runs before this, which will construct a false state
         if children > 0:
-            false_state = self.state_queue.pop()
-            # restore nat_language_indicator because it shouldn't of been emptied.
-            self.nat_language_indicator = false_state.nat_language_of_lists_indicator
             self.nat_language_indicator.append(children)
 
     def construct_mapping(
-        self, node: yaml.MappingNode, deep=False
+        self, node: yaml.MappingNode, deep=False, is_nat_language_map=False
     ) -> dict[Hashable, Any]:
         # This method will run for each map in a YamlObject.
         result = super().construct_mapping(node, deep)
@@ -113,7 +109,7 @@ def construct_mapping(
         children = self.count_children(result)
         result = parse_dict(result, new_translations_map, self.env)
 
-        if children > 0:
+        if children > 0 and not is_nat_language_map:
             new_state = State(
                 children, new_translations_map, self.nat_language_indicator
             )
@@ -229,7 +225,7 @@ def translate_translations_map(trans_map: dict, language: str) -> dict:
 def _natural_language_map_translation(
     loader: StateLoader, node: yaml.MappingNode
 ) -> Any:
-    result = loader.construct_mapping(node)
+    result = loader.construct_mapping(node, is_nat_language_map=True)
 
     all_types = {type(v) for v in result.values()}
     assert (
@@ -322,7 +318,6 @@ def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObje
     validate_pre_dsl(yaml_object)
 
     translated_yaml_ob = translate_yaml(yaml_stream, language)
-    print(f"result: {translated_yaml_ob}")
     translated_yaml_string = convert_to_yaml(translated_yaml_ob)
     _validate_dsl(_parse_yaml(translated_yaml_string))
     if to_file:

From 3f20d1083ff6ce905de519481cc10b55e305fd03 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 18 Mar 2025 18:35:59 +0100
Subject: [PATCH 073/111] Made attempt for stdout/stderr

---
 tested/dsl/schema-strict-nat-translation.json | 1304 +++++++++++++----
 tested/nat_translation.py                     |  273 +---
 tests/test_preprocess_dsl.py                  |  117 +-
 3 files changed, 1073 insertions(+), 621 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 751f5499..4f23a30f 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -49,11 +49,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -103,12 +117,26 @@
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "type" : "array",
-            "minItems" : 1,
-            "items" : {
-              "$ref" : "#/definitions/tab"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "type" : "array",
+                "minItems" : 1,
+                "items" : {
+                  "$ref" : "#/definitions/tab"
+                }
+              }
             }
           }
         }
@@ -124,12 +152,26 @@
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "type" : "array",
-            "minItems" : 1,
-            "items" : {
-              "$ref" : "#/definitions/unit"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "type" : "array",
+                "minItems" : 1,
+                "items" : {
+                  "$ref" : "#/definitions/unit"
+                }
+              }
             }
           }
         }
@@ -152,11 +194,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -172,9 +228,23 @@
               "type" : "string"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "string"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "string"
+                  }
+                }
               }
             }
           ],
@@ -232,11 +302,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -252,9 +336,23 @@
               "type" : "string"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "string"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "string"
+                  }
+                }
               }
             }
           ],
@@ -305,12 +403,26 @@
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "type" : "array",
-            "minItems" : 1,
-            "items" : {
-              "$ref" : "#/definitions/context"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "type" : "array",
+                "minItems" : 1,
+                "items" : {
+                  "$ref" : "#/definitions/context"
+                }
+              }
             }
           }
         }
@@ -326,12 +438,26 @@
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "type" : "array",
-            "minItems" : 1,
-            "items" : {
-              "$ref" : "#/definitions/case"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "type" : "array",
+                "minItems" : 1,
+                "items" : {
+                  "$ref" : "#/definitions/case"
+                }
+              }
             }
           }
         }
@@ -347,12 +473,26 @@
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "type" : "array",
-            "minItems" : 1,
-            "items" : {
-              "$ref" : "#/definitions/testcase"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "type" : "array",
+                "minItems" : 1,
+                "items" : {
+                  "$ref" : "#/definitions/testcase"
+                }
+              }
             }
           }
         }
@@ -368,12 +508,26 @@
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "type" : "array",
-            "minItems" : 1,
-            "items" : {
-              "$ref" : "#/definitions/script"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "type" : "array",
+                "minItems" : 1,
+                "items" : {
+                  "$ref" : "#/definitions/script"
+                }
+              }
             }
           }
         }
@@ -396,11 +550,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -436,11 +604,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -470,9 +652,23 @@
               "$ref" : "#/definitions/message"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/message"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/message"
+                  }
+                }
               }
             }
           ]
@@ -489,14 +685,28 @@
               ]
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : [
-                  "string",
-                  "number",
-                  "integer",
-                  "boolean"
-                ]
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : [
+                      "string",
+                      "number",
+                      "integer",
+                      "boolean"
+                    ]
+                  }
+                }
               }
             }
           ]
@@ -515,16 +725,30 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "type" : [
-                    "string",
-                    "number",
-                    "integer",
-                    "boolean"
-                  ]
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "type" : [
+                        "string",
+                        "number",
+                        "integer",
+                        "boolean"
+                      ]
+                    }
+                  }
                 }
               }
             }
@@ -546,9 +770,23 @@
               "$ref" : "#/definitions/exceptionChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/exceptionChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/exceptionChannel"
+                  }
+                }
               }
             }
           ]
@@ -563,11 +801,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -580,9 +832,23 @@
               "$ref" : "#/definitions/returnOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/returnOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/returnOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -595,9 +861,23 @@
               "$ref" : "#/definitions/textOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/textOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -609,9 +889,23 @@
               "$ref" : "#/definitions/textOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/textOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -623,9 +917,23 @@
               "$ref" : "#/definitions/fileOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/fileOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/fileOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -646,9 +954,23 @@
               "$ref" : "#/definitions/message"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/message"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/message"
+                  }
+                }
               }
             }
           ]
@@ -665,14 +987,28 @@
               ]
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : [
-                  "string",
-                  "number",
-                  "integer",
-                  "boolean"
-                ]
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : [
+                      "string",
+                      "number",
+                      "integer",
+                      "boolean"
+                    ]
+                  }
+                }
               }
             }
           ]
@@ -691,16 +1027,30 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "type" : [
-                    "string",
-                    "number",
-                    "integer",
-                    "boolean"
-                  ]
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "type" : [
+                        "string",
+                        "number",
+                        "integer",
+                        "boolean"
+                      ]
+                    }
+                  }
                 }
               }
             }
@@ -722,9 +1072,23 @@
               "$ref" : "#/definitions/exceptionChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/exceptionChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/exceptionChannel"
+                  }
+                }
               }
             }
           ]
@@ -739,11 +1103,25 @@
               }
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "type" : "array",
-                "items" : {
-                  "$ref" : "#/definitions/file"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/file"
+                    }
+                  }
                 }
               }
             }
@@ -756,9 +1134,23 @@
               "$ref" : "#/definitions/returnOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/returnOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/returnOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -771,9 +1163,23 @@
               "$ref" : "#/definitions/textOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/textOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -785,9 +1191,23 @@
               "$ref" : "#/definitions/textOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/textOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -799,9 +1219,23 @@
               "$ref" : "#/definitions/fileOutputChannel"
             },
             {
-              "type" : "natural_language",
-              "additionalProperties": {
-                "$ref" : "#/definitions/fileOutputChannel"
+              "type" : "object",
+              "required": [
+                "__tag__",
+                "value"
+              ],
+              "properties" : {
+                "__tag__": {
+                  "type" : "string",
+                  "description" : "The tag used in the yaml",
+                  "const":  "!natural_language"
+                },
+                "value":{
+                  "type": "object",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textOutputChannel"
+                  }
+                }
               }
             }
           ]
@@ -819,15 +1253,29 @@
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
-          "description" : "Programming-language-specific statement or expression.",
-          "type" : "programming_language",
-          "minProperties" : 1,
-          "propertyNames" : {
-            "$ref" : "#/definitions/programmingLanguage"
-          },
-          "items" : {
-            "type" : "string",
-            "description" : "A language-specific literal, which will be used verbatim."
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!programming_language"
+            },
+            "value":{
+              "type": "object",
+              "description" : "Programming-language-specific statement or expression.",
+              "minProperties" : 1,
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "string",
+                "description" : "A language-specific literal, which will be used verbatim."
+              }
+            }
           }
         }
       ]
@@ -839,21 +1287,49 @@
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
-          "description" : "Programming-language-specific statement or expression.",
-          "type" : "programming_language",
-          "minProperties" : 1,
-          "propertyNames" : {
-            "$ref" : "#/definitions/programmingLanguage"
-          },
-          "items" : {
-            "type" : "string",
-            "description" : "A language-specific literal, which will be used verbatim."
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!programming_language"
+            },
+            "value":{
+              "type": "object",
+              "description" : "Programming-language-specific statement or expression.",
+              "minProperties" : 1,
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "string",
+                "description" : "A language-specific literal, which will be used verbatim."
+              }
+            }
           }
         },
         {
-          "type" : "natural_language",
-          "additionalProperties": {
-            "$ref" : "#/definitions/expressionOrStatement"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!natural_language"
+            },
+            "value":{
+              "type": "object",
+              "additionalProperties": {
+                "$ref" : "#/definitions/expressionOrStatement"
+              }
+            }
           }
         }
       ]
@@ -864,9 +1340,23 @@
           "$ref" : "#/definitions/yamlValue"
         },
         {
-          "type" : "expression",
-          "format" : "tested-dsl-expression",
-          "description" : "An expression in Python-syntax."
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
+          ],
+          "properties" : {
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!expression"
+            },
+            "value":{
+              "type": "string",
+              "format" : "tested-dsl-expression",
+              "description" : "An expression in Python-syntax."
+            }
+          }
         }
       ]
     },
@@ -907,9 +1397,23 @@
                   "type" : "string"
                 },
                 {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "type" : "string"
+                  "type" : "object",
+                  "required": [
+                    "__tag__",
+                    "value"
+                  ],
+                  "properties" : {
+                    "__tag__": {
+                      "type" : "string",
+                      "description" : "The tag used in the yaml",
+                      "const":  "!natural_language"
+                    },
+                    "value":{
+                      "type": "object",
+                      "additionalProperties": {
+                        "type" : "string"
+                      }
+                    }
                   }
                 }
               ],
@@ -948,9 +1452,23 @@
                   "$ref" : "#/definitions/textualType"
                 },
                 {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "$ref" : "#/definitions/textualType"
+                  "type" : "object",
+                  "required": [
+                    "__tag__",
+                    "value"
+                  ],
+                  "properties" : {
+                    "__tag__": {
+                      "type" : "string",
+                      "description" : "The tag used in the yaml",
+                      "const":  "!natural_language"
+                    },
+                    "value":{
+                      "type": "object",
+                      "additionalProperties": {
+                        "$ref" : "#/definitions/textualType"
+                      }
+                    }
                   }
                 }
               ]
@@ -978,9 +1496,23 @@
                   "$ref" : "#/definitions/textualType"
                 },
                 {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "$ref" : "#/definitions/textualType"
+                  "type" : "object",
+                  "required": [
+                    "__tag__",
+                    "value"
+                  ],
+                  "properties" : {
+                    "__tag__": {
+                      "type" : "string",
+                      "description" : "The tag used in the yaml",
+                      "const":  "!natural_language"
+                    },
+                    "value":{
+                      "type": "object",
+                      "additionalProperties": {
+                        "$ref" : "#/definitions/textualType"
+                      }
+                    }
                   }
                 }
               ]
@@ -1095,180 +1627,292 @@
           "$ref" : "#/definitions/yamlValueOrPythonExpression"
         },
         {
-          "type" : "oracle",
-          "additionalProperties" : false,
-          "required" : [
+          "type" : "object",
+          "required": [
+            "__tag__",
             "value"
           ],
           "properties" : {
-            "oracle" : {
-              "const" : "builtin"
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!oracle"
             },
-            "value" : {
-              "oneOf" : [
-                {
-                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            "value":{
+              "type": "object",
+              "additionalProperties" : false,
+              "required" : [
+                "value"
+              ],
+              "properties" : {
+                "oracle" : {
+                  "const" : "builtin"
                 },
-                {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
-                  }
+                "value" : {
+                  "oneOf" : [
+                    {
+                      "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                    },
+                    {
+                      "type" : "object",
+                      "required": [
+                        "__tag__",
+                        "value"
+                      ],
+                      "properties" : {
+                        "__tag__": {
+                          "type" : "string",
+                          "description" : "The tag used in the yaml",
+                          "const":  "!natural_language"
+                        },
+                        "value":{
+                          "type": "object",
+                          "additionalProperties": {
+                            "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                          }
+                        }
+                      }
+                    }
+                  ]
                 }
-              ]
+              }
             }
           }
         },
         {
-          "type" : "oracle",
-          "additionalProperties" : false,
-          "required" : [
-            "value",
-            "oracle",
-            "file"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
           ],
           "properties" : {
-            "oracle" : {
-              "const" : "custom_check"
-            },
-            "value" : {
-              "oneOf" : [
-                {
-                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
-                },
-                {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
-                  }
-                }
-              ]
-            },
-            "file" : {
+            "__tag__": {
               "type" : "string",
-              "description" : "The path to the file containing the custom check function."
+              "description" : "The tag used in the yaml",
+              "const":  "!oracle"
             },
-            "name" : {
-              "type" : "string",
-              "description" : "The name of the custom check function.",
-              "default" : "evaluate"
-            },
-            "arguments" : {
-              "oneOf" : [
-                {
-                  "type" : "array",
-                  "items" : {
-                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
-                  }
+            "value":{
+              "type": "object",
+              "additionalProperties" : false,
+              "required" : [
+                "value",
+                "oracle",
+                "file"
+              ],
+              "properties" : {
+                "oracle" : {
+                  "const" : "custom_check"
                 },
-                {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "type" : "array",
-                    "items" : {
+                "value" : {
+                  "oneOf" : [
+                    {
                       "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                    },
+                    {
+                      "type" : "object",
+                      "required": [
+                        "__tag__",
+                        "value"
+                      ],
+                      "properties" : {
+                        "__tag__": {
+                          "type" : "string",
+                          "description" : "The tag used in the yaml",
+                          "const":  "!natural_language"
+                        },
+                        "value":{
+                          "type": "object",
+                          "additionalProperties": {
+                            "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                          }
+                        }
+                      }
                     }
+                  ]
+                },
+                "file" : {
+                  "type" : "string",
+                  "description" : "The path to the file containing the custom check function."
+                },
+                "name" : {
+                  "type" : "string",
+                  "description" : "The name of the custom check function.",
+                  "default" : "evaluate"
+                },
+                "arguments" : {
+                  "oneOf" : [
+                    {
+                      "type" : "array",
+                      "items" : {
+                        "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                      }
+                    },
+                    {
+                      "type" : "object",
+                      "required": [
+                        "__tag__",
+                        "value"
+                      ],
+                      "properties" : {
+                        "__tag__": {
+                          "type" : "string",
+                          "description" : "The tag used in the yaml",
+                          "const":  "!natural_language"
+                        },
+                        "value":{
+                          "type": "object",
+                          "additionalProperties": {
+                            "type" : "array",
+                            "items" : {
+                              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  ],
+                  "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
+                },
+                "languages": {
+                  "type" : "array",
+                  "description" : "Which programming languages are supported by this oracle.",
+                  "items" : {
+                    "$ref" : "#/definitions/programmingLanguage"
                   }
                 }
-              ],
-              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
-            },
-            "languages": {
-              "type" : "array",
-              "description" : "Which programming languages are supported by this oracle.",
-              "items" : {
-                "$ref" : "#/definitions/programmingLanguage"
               }
             }
           }
         },
         {
-          "type" : "oracle",
-          "additionalProperties" : false,
-          "required" : [
-            "oracle",
-            "functions"
+          "type" : "object",
+          "required": [
+            "__tag__",
+            "value"
           ],
           "properties" : {
-            "oracle" : {
-              "const" : "specific_check"
-            },
-            "functions" : {
-              "minProperties" : 1,
-              "description" : "Language mapping of oracle functions.",
-              "type" : "object",
-              "propertyNames" : {
-                "$ref" : "#/definitions/programmingLanguage"
-              },
-              "items" : {
-                "type" : "object",
-                "required" : [
-                  "file"
-                ],
-                "properties" : {
-                  "file" : {
-                    "type" : "string",
-                    "description" : "The path to the file containing the custom check function."
-                  },
-                  "name" : {
-                    "type" : "string",
-                    "description" : "The name of the custom check function.",
-                    "default" : "evaluate"
-                  }
-                }
-              }
+            "__tag__": {
+              "type" : "string",
+              "description" : "The tag used in the yaml",
+              "const":  "!oracle"
             },
-            "arguments" : {
-              "oneOf" : [
-                {
+            "value":{
+              "type": "object",
+              "additionalProperties" : false,
+              "required" : [
+                "oracle",
+                "functions"
+              ],
+              "properties" : {
+                "oracle" : {
+                  "const" : "specific_check"
+                },
+                "functions" : {
                   "minProperties" : 1,
-                  "description" : "Language mapping of oracle arguments.",
+                  "description" : "Language mapping of oracle functions.",
                   "type" : "object",
                   "propertyNames" : {
                     "$ref" : "#/definitions/programmingLanguage"
                   },
                   "items" : {
-                    "type" : "array",
-                    "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
-                    "items" : {
-                      "type" : "string",
-                      "description" : "A language-specific literal, which will be used verbatim."
+                    "type" : "object",
+                    "required" : [
+                      "file"
+                    ],
+                    "properties" : {
+                      "file" : {
+                        "type" : "string",
+                        "description" : "The path to the file containing the custom check function."
+                      },
+                      "name" : {
+                        "type" : "string",
+                        "description" : "The name of the custom check function.",
+                        "default" : "evaluate"
+                      }
                     }
                   }
                 },
-                {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "minProperties" : 1,
-                    "description" : "Language mapping of oracle arguments.",
-                    "type" : "object",
-                    "propertyNames" : {
-                      "$ref" : "#/definitions/programmingLanguage"
-                    },
-                    "items" : {
-                      "type" : "array",
-                      "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                "arguments" : {
+                  "oneOf" : [
+                    {
+                      "minProperties" : 1,
+                      "description" : "Language mapping of oracle arguments.",
+                      "type" : "object",
+                      "propertyNames" : {
+                        "$ref" : "#/definitions/programmingLanguage"
+                      },
                       "items" : {
-                        "type" : "string",
-                        "description" : "A language-specific literal, which will be used verbatim."
+                        "type" : "array",
+                        "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                        "items" : {
+                          "type" : "string",
+                          "description" : "A language-specific literal, which will be used verbatim."
+                        }
+                      }
+                    },
+                    {
+                      "type" : "object",
+                      "required": [
+                        "__tag__",
+                        "value"
+                      ],
+                      "properties" : {
+                        "__tag__": {
+                          "type" : "string",
+                          "description" : "The tag used in the yaml",
+                          "const":  "!natural_language"
+                        },
+                        "value":{
+                          "type": "object",
+                          "additionalProperties": {
+                            "minProperties" : 1,
+                            "description" : "Language mapping of oracle arguments.",
+                            "type" : "object",
+                            "propertyNames" : {
+                              "$ref" : "#/definitions/programmingLanguage"
+                            },
+                            "items" : {
+                              "type" : "array",
+                              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                              "items" : {
+                                "type" : "string",
+                                "description" : "A language-specific literal, which will be used verbatim."
+                              }
+                            }
+                          }
+                        }
                       }
                     }
-                  }
-                }
-              ]
-            },
-            "value" : {
-              "oneOf" : [
-                {
-                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  ]
                 },
-                {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
-                  }
+                "value" : {
+                  "oneOf" : [
+                    {
+                      "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                    },
+                    {
+                      "type" : "object",
+                      "required": [
+                        "__tag__",
+                        "value"
+                      ],
+                      "properties" : {
+                        "__tag__": {
+                          "type" : "string",
+                          "description" : "The tag used in the yaml",
+                          "const":  "!natural_language"
+                        },
+                        "value":{
+                          "type": "object",
+                          "additionalProperties": {
+                            "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                          }
+                        }
+                      }
+                    }
+                  ]
                 }
-              ]
+              }
             }
           }
         }
@@ -1305,9 +1949,23 @@
             "description" : {
               "oneOf": [
                 {
-                  "type" : "natural_language",
-                  "additionalProperties": {
-                    "type" : "string"
+                  "type" : "object",
+                  "required": [
+                    "__tag__",
+                    "value"
+                  ],
+                  "properties" : {
+                    "__tag__": {
+                      "type" : "string",
+                      "description" : "The tag used in the yaml",
+                      "const":  "!natural_language"
+                    },
+                    "value":{
+                      "type": "object",
+                      "additionalProperties": {
+                        "type" : "string"
+                      }
+                    }
                   }
                 },
                 {
@@ -1389,12 +2047,10 @@
     "yamlValue" : {
       "description" : "A value represented as YAML.",
       "not" : {
-        "type" : [
-          "oracle",
-          "expression",
-          "natural_language",
-          "programming_language"
-        ]
+        "properties": {
+          "__tag__": { "type": "string" }
+        },
+        "type": "object"
       }
     },
     "inheritableConfigObject": {
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 2e90ddc3..27ac5f3a 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,11 +1,12 @@
 import os
 import sys
-from collections import deque
+from collections import deque, OrderedDict
 from pathlib import Path
 from typing import Any, Hashable, cast
 
 import yaml
 from jinja2 import Environment
+from yaml.nodes import MappingNode, ScalarNode, SequenceNode
 
 from tested.datatypes import AllTypes
 from tested.dsl.translate_parser import (
@@ -28,6 +29,7 @@
 from tested.utils import get_args
 
 
+
 def validate_pre_dsl(dsl_object: YamlObject):
     """
     Validate a DSl object.
@@ -51,128 +53,59 @@ def validate_pre_dsl(dsl_object: YamlObject):
         message = "Validating the DSL resulted in some errors."
         raise ExceptionGroup(message, the_errors)
 
+class CustomDumper(yaml.SafeDumper):
 
-class State:
-    def __init__(
-        self, children: int, translations_map: dict, nat_language_indicator: list
-    ):
-        self.nat_language_of_lists_indicator = nat_language_indicator
-        self.translations_map = translations_map
-        self.total_children = sum(self.nat_language_of_lists_indicator)
-
-        if children > len(self.nat_language_of_lists_indicator):
-            self.total_children += children - len(self.nat_language_of_lists_indicator)
-
-    def is_finished(self) -> bool:
-        return self.total_children == 0
-
-
-class StateLoader(yaml.SafeLoader):
-    def __init__(self, stream):
-        super().__init__(stream)
-        self.lang = ""
-        self.state_queue: deque[State] = deque()
-        self.nat_language_indicator = []
-        self.env = create_enviroment()
+    def represent_with_tag(self, tag, value):
+        if isinstance(value, dict):
+            return self.represent_mapping(tag, value)
+        elif isinstance(value, list):
+            return self.represent_sequence(tag, value)
+        else:
+            return self.represent_scalar(tag, value)
 
     @staticmethod
-    def count_children(dictionary: dict):
-        # The children of a dictionary will always be stored into a list that will
-        # be empty at the moment it is being parsed.
-        return sum(1 for v in dictionary.values() if isinstance(v, list) and not v)
-
-    def set_language(self, lang: str):
-        self.lang = lang
-
-    def add_nat_language_indication(self, children: int):
-
-        if children > 0:
-            self.nat_language_indicator.append(children)
-
-    def construct_mapping(
-        self, node: yaml.MappingNode, deep=False, is_nat_language_map=False
-    ) -> dict[Hashable, Any]:
-        # This method will run for each map in a YamlObject.
-        result = super().construct_mapping(node, deep)
-        new_translations_map = (
-            self.state_queue[0].translations_map if self.state_queue else {}
-        )
-
-        if "translations" in result:
-            new_translations_map = flatten_stack(
-                [
-                    new_translations_map,
-                    translate_translations_map(result.pop("translations"), self.lang),
-                ]
-            )
-
-        children = self.count_children(result)
-        result = parse_dict(result, new_translations_map, self.env)
-
-        if children > 0 and not is_nat_language_map:
-            new_state = State(
-                children, new_translations_map, self.nat_language_indicator
-            )
-            self.state_queue.append(new_state)
-            self.nat_language_indicator = []
-
-        return result
-
-    def construct_sequence(self, node: yaml.SequenceNode, deep=False) -> list[Any]:
-        result = super().construct_sequence(node, deep)
-
-        translations_map = (
-            self.state_queue[0].translations_map if self.state_queue else {}
+    def custom_representer(dumper, data):
+        if '__tag__' in data:
+            if data['__tag__'] != "!programming_language":
+                return dumper.represent_with_tag(data['__tag__'], data['value'])
+            else:
+                data = data['value']
+
+        return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data)
+
+
+def construct_custom(loader, tag_suffix, node):
+    if isinstance(node, MappingNode):
+        data = loader.construct_mapping(node)
+    elif isinstance(node, ScalarNode):
+        data = loader.construct_scalar(node)
+    elif isinstance(node, SequenceNode):
+        data = loader.construct_sequence(node)
+    else:
+        raise yaml.constructor.ConstructorError(
+            None, None,
+            f"expected a mapping, scalar, or sequence node, but found {node.id}",
+            node.start_mark
         )
 
-        result = parse_list(result, translations_map, self.env)
+    return {'__tag__': tag_suffix, 'value': data}
 
-        if len(self.state_queue) > 0:
-            self.state_queue[0].total_children -= 1
-            if self.state_queue[0].is_finished():
-                self.state_queue.popleft()
+def translate_yaml(data: Any, translations:dict, language:str, env: Environment):
+    if isinstance(data, dict):
+        if '__tag__' in data and data['__tag__'] == '!natural_language':
+            return translate_yaml(data['value'][language], translations, language, env)
 
-        return result
+        current_translations = data.pop('translations', {})
+        for key, value in current_translations.items():
+            current_translations[key] = value[language]
+        merged_translations = {**translations, **current_translations}
 
-
-def parse_value(
-    value: YamlObject, flattened_stack: dict, env: Environment
-) -> YamlObject:
-
-    if isinstance(value, str):
-        return type(value)(format_string(value, flattened_stack, env))
-
-    return value
-
-
-def parse_list(value: list[Any], flattened_stack: dict, env: Environment) -> list[Any]:
-    if len(value) > 0:
-        return [parse_value(v, flattened_stack, env) for v in value]
-    return value
-
-
-def parse_dict(
-    value: dict[Hashable, Any], flattened_stack: dict, env: Environment
-) -> dict[Hashable, Any]:
-    return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
-
-
-def flatten_stack(translation_stack: list) -> dict:
-    # Will transform a list of translation maps into a dict that
-    # has all the keys defined over all the different translation map and will have
-    # the value of the newest definition. In this definition we also chose
-    # the translation of the provided language.
-    flattened = {}
-    for d in translation_stack:
-        flattened.update({k: v for k, v in d.items()})
-    return flattened
-
-
-def format_string(string: str, translations: dict, env: Environment) -> str:
-    template = env.from_string(string)
-    result = template.render(translations)
-
-    return result
+        return {key: translate_yaml(value, merged_translations, language, env) for key, value in data.items()}
+    elif isinstance(data, list):
+        return [translate_yaml(item, translations, language, env) for item in data]
+    elif isinstance(data, str):
+        return env.from_string(data).render(translations)
+    return data
 
 
 def wrap_in_braces(value):
@@ -193,94 +126,9 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
         yaml_file.write(yaml_string)
 
 
-def convert_to_yaml(yaml_object: YamlObject) -> str:
-    def oracle_representer(dumper, data):
-        return dumper.represent_mapping("!oracle", data)
-
-    def expression_representer(dumper, data):
-        return dumper.represent_scalar("!expression", data)
-
-    def programming_language_map_representer(dumper, data):
-        return dumper.represent_mapping("tag:yaml.org,2002:map", dict(data))
-
-    yaml.add_representer(ReturnOracle, oracle_representer)
-    yaml.add_representer(ExpressionString, expression_representer)
-    yaml.add_representer(ProgrammingLanguageMap, programming_language_map_representer)
-    return yaml.dump(yaml_object, sort_keys=False)
-
-
-def translate_map(value: YamlObject, language: str):
-    assert isinstance(
-        value, dict
-    ), "The translation map does not consist of dictionaries."
-    assert language in value
-    value = value[language]
-    return value
-
-
-def translate_translations_map(trans_map: dict, language: str) -> dict:
-    return {k: translate_map(v, language) for k, v in trans_map.items()}
-
-
-def _natural_language_map_translation(
-    loader: StateLoader, node: yaml.MappingNode
-) -> Any:
-    result = loader.construct_mapping(node, is_nat_language_map=True)
-
-    all_types = {type(v) for v in result.values()}
-    assert (
-        len(all_types) == 1
-    ), "A natural language map should have values with the same type."
-    children = loader.count_children(result)
-    loader.add_nat_language_indication(children)
-    return result[loader.lang]
-
-
-def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
-    result = _parse_yaml_value(loader, node)
-    assert isinstance(
-        result, dict
-    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
-    return NaturalLanguageMap(result)
-
-
-def _programming_language_map(
-    loader: yaml.Loader, node: yaml.Node
-) -> ProgrammingLanguageMap:
-    result = _parse_yaml_value(loader, node)
-    assert isinstance(
-        result, dict
-    ), f"A programming language map must be an object, got {result} which is a {type(result)}."
-    return ProgrammingLanguageMap(result)
-
-
-def dict_trans(loader: StateLoader, node: yaml.MappingNode) -> dict[Hashable, Any]:
-    return loader.construct_mapping(node)
-
-
-def translate_yaml(yaml_stream: str, language: str) -> YamlObject:
-    """
-    Parse a string or stream to YAML.
-    """
-    try:
-        loader = StateLoader(yaml_stream)
-        loader.set_language(language)
-        for types in get_args(AllTypes):
-            for actual_type in types:
-                loader.add_constructor("!" + actual_type, _custom_type_constructors)
-        loader.add_constructor("!expression", _expression_string)
-        loader.add_constructor("!oracle", _return_oracle)
-        loader.add_constructor("!natural_language", _natural_language_map_translation)
-        loader.add_constructor("!programming_language", _programming_language_map)
-        # This line is need because otherwise there won't
-        # be a full translations map on time.
-        loader.add_constructor(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_trans
-        )
-
-        return loader.get_data()
-    except yaml.MarkedYAMLError as exc:
-        raise_yaml_error(yaml_stream, exc)
+def convert_to_yaml(translated_data: Any) -> str:
+    CustomDumper.add_representer(dict, CustomDumper.custom_representer)
+    return yaml.dump(translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False)
 
 
 def parse_yaml(yaml_stream: str) -> YamlObject:
@@ -288,13 +136,7 @@ def parse_yaml(yaml_stream: str) -> YamlObject:
     Parse a string or stream to YAML.
     """
     loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.CSafeLoader)
-    for types in get_args(AllTypes):
-        for actual_type in types:
-            yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
-    yaml.add_constructor("!expression", _expression_string, loader)
-    yaml.add_constructor("!oracle", _return_oracle, loader)
-    yaml.add_constructor("!natural_language", _natural_language_map, loader)
-    yaml.add_constructor("!programming_language", _programming_language_map, loader)
+    yaml.add_multi_constructor('', construct_custom, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
@@ -317,13 +159,16 @@ def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObje
     yaml_object = parse_yaml(yaml_stream)
     validate_pre_dsl(yaml_object)
 
-    translated_yaml_ob = translate_yaml(yaml_stream, language)
-    translated_yaml_string = convert_to_yaml(translated_yaml_ob)
-    _validate_dsl(_parse_yaml(translated_yaml_string))
+    enviroment = create_enviroment()
+    translated_data = translate_yaml(yaml_object, {}, language, enviroment)
+
+    translated_yaml_string = convert_to_yaml(translated_data)
+    yaml_object = _parse_yaml(translated_yaml_string)
+    _validate_dsl(yaml_object)
     if to_file:
         generate_new_yaml(path, translated_yaml_string, language)
 
-    return translated_yaml_ob
+    return yaml_object
 
 
 if __name__ == "__main__":
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 7a72c334..bf68b8f2 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -11,8 +11,6 @@
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
-    parse_dict,
-    parse_list,
     parse_yaml,
     run_translation,
     translate_yaml,
@@ -51,11 +49,11 @@
             en: 'The {{result}} is 10'
             nl: 'Het {{result}} is 10'
         - expression: !natural_language
-            en: !expression "count"
-            nl: !expression "tellen"
+            en: "count"
+            nl: "tellen"
           return: !natural_language
-            en: 'count'
-            nl: 'tellen'
+            en: !expression 'count'
+            nl: !expression 'tellen'
         - expression: 'ok(10)'
           return: !oracle
             value: !natural_language
@@ -117,15 +115,15 @@
             - statement: '{{result}} = Trying(10)'
             - expression: 'count_words({{result}})'
               return: 'The {{result}} is 10'
-            - expression: !expression "count"
-              return: 'count'
+            - expression: "count"
+              return: !expression 'count'
         nl:
           - testcases:
             - statement: '{{result}} = Proberen(10)'
             - expression: 'tel_woorden({{result}})'
               return: 'Het {{result}} is 10'
-            - expression: !expression "tellen"
-              return: 'tellen'
+            - expression: "tellen"
+              return: !expression 'tellen'
   - tab: "task2"
     contexts:
         - testcases: !natural_language
@@ -133,28 +131,28 @@
               - statement: '{{result}} = Trying(10)'
               - expression: 'count_words({{result}})'
                 return: 'The {{result}} is 10'
-              - expression: !expression "count"
-                return: 'count'
+              - expression: "count"
+                return: !expression 'count'
             nl:
               - statement: '{{result}} = Proberen(10)'
               - expression: 'tel_woorden({{result}})'
                 return: 'Het {{result}} is 10'
-              - expression: !expression "tellen"
-                return: 'tellen'
+              - expression: "tellen"
+                return: !expression 'tellen'
   - tab: "task3"
     testcases: !natural_language
         en:
           - statement: '{{result}} = Trying(10)'
           - expression: 'count_words({{result}})'
             return: 'The {{result}} is 10'
-          - expression: !expression "count"
-            return: 'count'
+          - expression: "count"
+            return: !expression 'count'
         nl:
           - statement: '{{result}} = Proberen(10)'
           - expression: 'tel_woorden({{result}})'
             return: 'Het {{result}} is 10'
-          - expression: !expression "tellen"
-            return: 'tellen'
+          - expression: "tellen"
+            return: !expression 'tellen'
 """.strip()
 
 
@@ -169,8 +167,8 @@ def test_natural_translate_unit_test():
     - statement: results_context = Trying(10)
     - expression: count_words(results_context)
       return: The results_context is 10
-    - expression: !expression 'count'
-      return: count
+    - expression: count
+      return: !expression 'count'
     - expression: ok(10)
       return: !oracle
         value: The results_context 10 is OK!
@@ -212,25 +210,27 @@ def test_natural_translate_unit_test():
     - statement: results = Trying(10)
     - expression: count_words(results)
       return: The results is 10
-    - expression: !expression 'count'
-      return: count
+    - expression: count
+      return: !expression 'count'
 - tab: task2
   contexts:
   - testcases:
     - statement: results = Trying(10)
     - expression: count_words(results)
       return: The results is 10
-    - expression: !expression 'count'
-      return: count
+    - expression: count
+      return: !expression 'count'
 - tab: task3
   testcases:
   - statement: results = Trying(10)
   - expression: count_words(results)
     return: The results is 10
-  - expression: !expression 'count'
-    return: count
+  - expression: count
+    return: !expression 'count'
 """.strip()
-    translated_dsl = translate_yaml(test_unit_yaml_str, "en")
+    enviroment = create_enviroment()
+    yaml_object = parse_yaml(test_unit_yaml_str)
+    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
     translated_yaml = convert_to_yaml(translated_dsl)
     assert translated_yaml.strip() == translated_yaml_str
 
@@ -340,44 +340,20 @@ def test_natural_translate_io_test():
   - expression: tests(11)
     return: 11
 """.strip()
-    translated_dsl = translate_yaml(yaml_str, "en")
+    enviroment = create_enviroment()
+    yaml_object = parse_yaml(yaml_str)
+    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
     translated_yaml = convert_to_yaml(translated_dsl)
     assert translated_yaml.strip() == translated_yaml_str
 
 
-def test_translate_parse():
-    env = create_enviroment()
-    flattened_stack = {"human": "mens", "number": "getal"}
-    value = {
-        "key1": "value1_{{human}}",
-        "key2": "value2_{{number}}",
-        "key3": 10,
-    }
-    expected_value = {
-        "key1": "value1_mens",
-        "key2": "value2_getal",
-        "key3": 10,
-    }
-    parsed_result = parse_dict(cast(dict[Hashable, Any], value), flattened_stack, env)
-    assert parsed_result == expected_value
-
-    value = ["value1_{{human}}", "value2_{{number}}", 10]
-    expected_value = ["value1_mens", "value2_getal", 10]
-    parsed_result = parse_list(value, flattened_stack, env)
-    assert parsed_result == expected_value
-
-    value = []
-    expected_value = []
-    parsed_result = parse_list(value, flattened_stack, env)
-    assert parsed_result == expected_value
-
-
 def test_validation():
     yaml_object = parse_yaml(test_unit_yaml_str)
     validate_pre_dsl(yaml_object)
 
-    translated_yaml_ob = translate_yaml(test_unit_yaml_str, "en")
-    translated_yaml_string = convert_to_yaml(translated_yaml_ob)
+    enviroment = create_enviroment()
+    translated_data = translate_yaml(yaml_object, {}, "en", enviroment)
+    translated_yaml_string = convert_to_yaml(translated_data)
     _validate_dsl(_parse_yaml(translated_yaml_string))
 
 
@@ -406,31 +382,6 @@ def test_wrong_natural_translation_suite():
         assert False, "Expected ExceptionGroup error"
 
 
-def test_yaml_with_syntax_error():
-    yaml_str = """
-tabs:
-- tab: animals
-  testcases:
-  - expression: tests(11)
-    return: 11
-  - expression: !programming_language
-      javascript: animals_javascript(1 + 1)
-      typescript: animals_typescript(1 + 1)
-      java: Submission.animals_java(1 + 1)
-      python: !nat_language
-        en: animals_python_en(1 + 1)
-        nl: animals_python_nl(1 + 1)
-    return: 2
-    """.strip()
-
-    try:
-        parse_yaml(yaml_str)
-    except yaml.MarkedYAMLError:
-        print("As expected")
-    else:
-        assert False, "Expected yaml.MarkedYAMLError error"
-
-
 def test_run_is_correct(mocker: MockerFixture):
     s = mocker.spy(tested.nat_translation, name="generate_new_yaml")  # type: ignore[reportAttributeAccessIssue]
     mock_files = [
@@ -473,7 +424,7 @@ def test_run_is_correct_when_no_file():
 
     try:
         run_translation(Path("suite.yaml"), "en", False)
-    except FileNotFoundError as e:
+    except FileNotFoundError:
         print("As expected")
     else:
         assert False, "Expected FileNotFoundError error"

From 7952b2d537173973f51871d01b4c1033126851c2 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 20 Mar 2025 11:44:53 +0100
Subject: [PATCH 074/111] made new version for translations

---
 tested/nat_translation.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 27ac5f3a..cb046504 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -75,19 +75,22 @@ def custom_representer(dumper, data):
 
 
 def construct_custom(loader, tag_suffix, node):
-    if isinstance(node, MappingNode):
-        data = loader.construct_mapping(node)
-    elif isinstance(node, ScalarNode):
-        data = loader.construct_scalar(node)
-    elif isinstance(node, SequenceNode):
-        data = loader.construct_sequence(node)
-    else:
+
+    type2method = {
+        MappingNode: loader.construct_mapping,
+        ScalarNode: loader.construct_scalar,
+        SequenceNode: loader.construct_sequence
+    }
+
+    if not type(node) in type2method:
         raise yaml.constructor.ConstructorError(
             None, None,
             f"expected a mapping, scalar, or sequence node, but found {node.id}",
             node.start_mark
         )
 
+    data = type2method[type(node)](node)
+
     return {'__tag__': tag_suffix, 'value': data}
 
 def translate_yaml(data: Any, translations:dict, language:str, env: Environment):
@@ -131,7 +134,7 @@ def convert_to_yaml(translated_data: Any) -> str:
     return yaml.dump(translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False)
 
 
-def parse_yaml(yaml_stream: str) -> YamlObject:
+def parse_yaml(yaml_stream: str):
     """
     Parse a string or stream to YAML.
     """
@@ -163,6 +166,7 @@ def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObje
     translated_data = translate_yaml(yaml_object, {}, language, enviroment)
 
     translated_yaml_string = convert_to_yaml(translated_data)
+    #TODO: FIX redundancy
     yaml_object = _parse_yaml(translated_yaml_string)
     _validate_dsl(yaml_object)
     if to_file:

From 3e9f27a8a967fd9a459e254ece8b7e68bd4b0188 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 20 Mar 2025 20:34:39 +0100
Subject: [PATCH 075/111] used a different way to convert to yamlObject

---
 tested/nat_translation.py | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index cb046504..b2ce4541 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -110,6 +110,19 @@ def translate_yaml(data: Any, translations:dict, language:str, env: Environment)
         return env.from_string(data).render(translations)
     return data
 
+def to_yaml_object(data: Any) -> YamlObject:
+    if isinstance(data, dict):
+        if '__tag__' in data:
+            value = data['value']
+            if data['__tag__'] == '!oracle':
+                return ReturnOracle(to_yaml_object(value))
+            if data['__tag__'] == '!expression':
+                return ExpressionString(to_yaml_object(value))
+            return value
+    elif isinstance(data, list):
+        return [to_yaml_object(value) for value in data]
+
+    return data
 
 def wrap_in_braces(value):
     return f"{{{value}}}"
@@ -159,20 +172,20 @@ def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObje
         raise e
     _, ext = os.path.splitext(path)
     assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}."
-    yaml_object = parse_yaml(yaml_stream)
-    validate_pre_dsl(yaml_object)
+    parsed_yaml = parse_yaml(yaml_stream)
+    validate_pre_dsl(parsed_yaml)
 
     enviroment = create_enviroment()
-    translated_data = translate_yaml(yaml_object, {}, language, enviroment)
+    translated_data = translate_yaml(parsed_yaml, {}, language, enviroment)
 
-    translated_yaml_string = convert_to_yaml(translated_data)
-    #TODO: FIX redundancy
-    yaml_object = _parse_yaml(translated_yaml_string)
-    _validate_dsl(yaml_object)
     if to_file:
+        translated_yaml_string = convert_to_yaml(translated_data)
         generate_new_yaml(path, translated_yaml_string, language)
-
-    return yaml_object
+        return {}
+    else:
+        yaml_object = to_yaml_object(translated_data)
+        _validate_dsl(yaml_object)
+        return yaml_object
 
 
 if __name__ == "__main__":

From fc6dbc536e9547c7fa79ea60344011df435b754f Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 11:38:55 +0100
Subject: [PATCH 076/111] cleaned up code

---
 tested/nat_translation.py    | 77 ++++++++++++++++++++----------------
 tests/test_preprocess_dsl.py | 10 ++---
 2 files changed, 45 insertions(+), 42 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b2ce4541..41b71382 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,33 +1,22 @@
 import os
 import sys
-from collections import deque, OrderedDict
 from pathlib import Path
-from typing import Any, Hashable, cast
+from typing import Any, cast
 
 import yaml
 from jinja2 import Environment
 from yaml.nodes import MappingNode, ScalarNode, SequenceNode
 
-from tested.datatypes import AllTypes
 from tested.dsl.translate_parser import (
     DslValidationError,
     ExpressionString,
-    NaturalLanguageMap,
-    ProgrammingLanguageMap,
     ReturnOracle,
     YamlObject,
-    _custom_type_constructors,
-    _expression_string,
-    _parse_yaml,
-    _parse_yaml_value,
-    _return_oracle,
     _validate_dsl,
     convert_validation_error_to_group,
     load_schema_validator,
     raise_yaml_error,
 )
-from tested.utils import get_args
-
 
 
 def validate_pre_dsl(dsl_object: YamlObject):
@@ -53,6 +42,7 @@ def validate_pre_dsl(dsl_object: YamlObject):
         message = "Validating the DSL resulted in some errors."
         raise ExceptionGroup(message, the_errors)
 
+
 class CustomDumper(yaml.SafeDumper):
 
     def represent_with_tag(self, tag, value):
@@ -65,13 +55,15 @@ def represent_with_tag(self, tag, value):
 
     @staticmethod
     def custom_representer(dumper, data):
-        if '__tag__' in data:
-            if data['__tag__'] != "!programming_language":
-                return dumper.represent_with_tag(data['__tag__'], data['value'])
+        if "__tag__" in data:
+            if data["__tag__"] != "!programming_language":
+                return dumper.represent_with_tag(data["__tag__"], data["value"])
             else:
-                data = data['value']
+                data = data["value"]
 
-        return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data)
+        return dumper.represent_mapping(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data
+        )
 
 
 def construct_custom(loader, tag_suffix, node):
@@ -79,44 +71,54 @@ def construct_custom(loader, tag_suffix, node):
     type2method = {
         MappingNode: loader.construct_mapping,
         ScalarNode: loader.construct_scalar,
-        SequenceNode: loader.construct_sequence
+        SequenceNode: loader.construct_sequence,
     }
 
     if not type(node) in type2method:
         raise yaml.constructor.ConstructorError(
-            None, None,
+            None,
+            None,
             f"expected a mapping, scalar, or sequence node, but found {node.id}",
-            node.start_mark
+            node.start_mark,
         )
 
     data = type2method[type(node)](node)
 
-    return {'__tag__': tag_suffix, 'value': data}
+    return {"__tag__": tag_suffix, "value": data}
+
 
-def translate_yaml(data: Any, translations:dict, language:str, env: Environment):
+def translate_yaml(
+    data: Any, translations: dict, language: str, env: Environment
+) -> Any:
     if isinstance(data, dict):
-        if '__tag__' in data and data['__tag__'] == '!natural_language':
-            return translate_yaml(data['value'][language], translations, language, env)
+        if "__tag__" in data and data["__tag__"] == "!natural_language":
+            return translate_yaml(data["value"][language], translations, language, env)
 
-        current_translations = data.pop('translations', {})
+        current_translations = data.pop("translations", {})
         for key, value in current_translations.items():
             current_translations[key] = value[language]
         merged_translations = {**translations, **current_translations}
 
-        return {key: translate_yaml(value, merged_translations, language, env) for key, value in data.items()}
+        return {
+            key: translate_yaml(value, merged_translations, language, env)
+            for key, value in data.items()
+        }
     elif isinstance(data, list):
         return [translate_yaml(item, translations, language, env) for item in data]
     elif isinstance(data, str):
         return env.from_string(data).render(translations)
     return data
 
+
 def to_yaml_object(data: Any) -> YamlObject:
     if isinstance(data, dict):
-        if '__tag__' in data:
-            value = data['value']
-            if data['__tag__'] == '!oracle':
-                return ReturnOracle(to_yaml_object(value))
-            if data['__tag__'] == '!expression':
+        if "__tag__" in data:
+            value = data["value"]
+            if data["__tag__"] == "!oracle":
+                result = to_yaml_object(value)
+                assert isinstance(result, dict)
+                return ReturnOracle(result)
+            if data["__tag__"] == "!expression":
                 return ExpressionString(to_yaml_object(value))
             return value
     elif isinstance(data, list):
@@ -124,6 +126,7 @@ def to_yaml_object(data: Any) -> YamlObject:
 
     return data
 
+
 def wrap_in_braces(value):
     return f"{{{value}}}"
 
@@ -142,17 +145,21 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
         yaml_file.write(yaml_string)
 
 
+CustomDumper.add_representer(dict, CustomDumper.custom_representer)
+
+
 def convert_to_yaml(translated_data: Any) -> str:
-    CustomDumper.add_representer(dict, CustomDumper.custom_representer)
-    return yaml.dump(translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False)
+    return yaml.dump(
+        translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False
+    )
 
 
-def parse_yaml(yaml_stream: str):
+def parse_yaml(yaml_stream: str) -> Any:
     """
     Parse a string or stream to YAML.
     """
     loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.CSafeLoader)
-    yaml.add_multi_constructor('', construct_custom, loader)
+    yaml.add_multi_constructor("", construct_custom, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index bf68b8f2..e82d3602 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,9 +1,5 @@
 from pathlib import Path
-from typing import Any, Hashable, cast
-from unittest.mock import ANY
 
-import pytest
-import yaml
 from pytest_mock import MockerFixture
 
 import tested
@@ -228,9 +224,9 @@ def test_natural_translate_unit_test():
   - expression: count
     return: !expression 'count'
 """.strip()
-    enviroment = create_enviroment()
-    yaml_object = parse_yaml(test_unit_yaml_str)
-    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
+    environment = create_enviroment()
+    parsed_yaml = parse_yaml(test_unit_yaml_str)
+    translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
     translated_yaml = convert_to_yaml(translated_dsl)
     assert translated_yaml.strip() == translated_yaml_str
 

From cbcb263622664d30a6a9717c7b3c0de94d60d96a Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 12:11:17 +0100
Subject: [PATCH 077/111] test why it test don't work in github

---
 tested/nat_translation.py    | 4 +---
 tests/test_preprocess_dsl.py | 1 +
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 41b71382..c20b3fa9 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -145,10 +145,8 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
         yaml_file.write(yaml_string)
 
 
-CustomDumper.add_representer(dict, CustomDumper.custom_representer)
-
-
 def convert_to_yaml(translated_data: Any) -> str:
+    CustomDumper.add_representer(dict, CustomDumper.custom_representer)
     return yaml.dump(
         translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False
     )
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index e82d3602..0efa43db 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -226,6 +226,7 @@ def test_natural_translate_unit_test():
 """.strip()
     environment = create_enviroment()
     parsed_yaml = parse_yaml(test_unit_yaml_str)
+    print(parsed_yaml)
     translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
     translated_yaml = convert_to_yaml(translated_dsl)
     assert translated_yaml.strip() == translated_yaml_str

From 16f55273e9c14b5ef92003bf9e13a5b0e943d707 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 13:08:34 +0100
Subject: [PATCH 078/111] adding more prints

---
 .github/workflows/ci.yml  | 1 +
 tested/nat_translation.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9cc37a9a..63c0b377 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,6 +17,7 @@ jobs:
     needs: [build_devcontainer]
     runs-on: ubuntu-latest
     steps:
+      - run: python -c "import yaml; print(yaml.__version__)"
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.ref }}
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index c20b3fa9..323a2401 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -73,7 +73,7 @@ def construct_custom(loader, tag_suffix, node):
         ScalarNode: loader.construct_scalar,
         SequenceNode: loader.construct_sequence,
     }
-
+    print(tag_suffix)
     if not type(node) in type2method:
         raise yaml.constructor.ConstructorError(
             None,

From f4c55d715b4762f985d9c166eefde36bdb5eaf06 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 13:43:42 +0100
Subject: [PATCH 079/111] adding more prints

---
 .github/workflows/ci.yml  | 1 -
 tested/nat_translation.py | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 63c0b377..9cc37a9a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -17,7 +17,6 @@ jobs:
     needs: [build_devcontainer]
     runs-on: ubuntu-latest
     steps:
-      - run: python -c "import yaml; print(yaml.__version__)"
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.ref }}
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 323a2401..03160920 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -156,7 +156,8 @@ def parse_yaml(yaml_stream: str) -> Any:
     """
     Parse a string or stream to YAML.
     """
-    loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.CSafeLoader)
+    print(yaml_stream)
+    loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader)
     yaml.add_multi_constructor("", construct_custom, loader)
 
     try:

From d9c115bffd0b2902af0ab3d803545506371d3563 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 13:48:00 +0100
Subject: [PATCH 080/111] trying something else

---
 tested/nat_translation.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 03160920..69e6de9f 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -157,11 +157,10 @@ def parse_yaml(yaml_stream: str) -> Any:
     Parse a string or stream to YAML.
     """
     print(yaml_stream)
-    loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader)
-    yaml.add_multi_constructor("", construct_custom, loader)
+    yaml.SafeLoader.add_multi_constructor("", construct_custom)
 
     try:
-        return yaml.load(yaml_stream, loader)
+        return yaml.safe_load(yaml_stream)
     except yaml.MarkedYAMLError as exc:
         raise_yaml_error(yaml_stream, exc)
 

From 77fbf13c8d94d10b9e35ccb2c85b92757b9d9494 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 14:50:08 +0100
Subject: [PATCH 081/111] revert back

---
 tested/nat_translation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 69e6de9f..03160920 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -157,10 +157,11 @@ def parse_yaml(yaml_stream: str) -> Any:
     Parse a string or stream to YAML.
     """
     print(yaml_stream)
-    yaml.SafeLoader.add_multi_constructor("", construct_custom)
+    loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader)
+    yaml.add_multi_constructor("", construct_custom, loader)
 
     try:
-        return yaml.safe_load(yaml_stream)
+        return yaml.load(yaml_stream, loader)
     except yaml.MarkedYAMLError as exc:
         raise_yaml_error(yaml_stream, exc)
 

From 35cf4151f4c34de7b3b2df7f09ccdfbc50b0092f Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 16:24:54 +0100
Subject: [PATCH 082/111] added test and cleaned up some of the code

---
 tested/dsl/translate_parser.py | 37 ++++------------------------------
 tested/nat_translation.py      |  3 +--
 tests/test_preprocess_dsl.py   | 19 +++++++++++++++++
 3 files changed, 24 insertions(+), 35 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 5e51018e..f1298fd5 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -88,27 +88,9 @@ class ReturnOracle(dict):
     pass
 
 
-class NaturalLanguageMap(dict):
-    pass
-
-
-class ProgrammingLanguageMap(dict):
-    pass
-
-
 OptionDict = dict[str, int | bool]
 YamlObject = (
-    YamlDict
-    | list
-    | bool
-    | float
-    | int
-    | str
-    | None
-    | ExpressionString
-    | ReturnOracle
-    | NaturalLanguageMap
-    | ProgrammingLanguageMap
+    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle
 )
 
 
@@ -209,14 +191,6 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
 
-def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
-    return isinstance(instance, NaturalLanguageMap)
-
-
-def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
-    return isinstance(instance, ProgrammingLanguageMap)
-
-
 def test(value: object) -> bool:
     if not isinstance(value, str):
         return False
@@ -235,12 +209,9 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         schema_object = json.load(schema_file)
 
     original_validator: Type[Validator] = validator_for(schema_object)
-    type_checker = (
-        original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
-        .redefine("expression", is_expression)
-        .redefine("natural_language", is_natural_language_map)
-        .redefine("programming_language", is_programming_language_map)
-    )
+    type_checker = original_validator.TYPE_CHECKER.redefine(
+        "oracle", is_oracle
+    ).redefine("expression", is_expression)
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 03160920..117769ae 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -73,7 +73,7 @@ def construct_custom(loader, tag_suffix, node):
         ScalarNode: loader.construct_scalar,
         SequenceNode: loader.construct_sequence,
     }
-    print(tag_suffix)
+
     if not type(node) in type2method:
         raise yaml.constructor.ConstructorError(
             None,
@@ -156,7 +156,6 @@ def parse_yaml(yaml_stream: str) -> Any:
     """
     Parse a string or stream to YAML.
     """
-    print(yaml_stream)
     loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader)
     yaml.add_multi_constructor("", construct_custom, loader)
 
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 0efa43db..8e5b4783 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -1,5 +1,6 @@
 from pathlib import Path
 
+import yaml
 from pytest_mock import MockerFixture
 
 import tested
@@ -417,6 +418,24 @@ def test_run_is_correct(mocker: MockerFixture):
     }
 
 
+def test_parsing_failed():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  expression: calculator(1 + 1)
+  - return: 2
+        """.strip()
+    try:
+        parse_yaml(yaml_str)
+    except yaml.MarkedYAMLError:
+        print("As expected")
+    else:
+        assert False, "Expected MarkedYAMLError error"
+
+
 def test_run_is_correct_when_no_file():
 
     try:

From d2fb379a85d1b2e96e305b8a0a830eefb33d04ee Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 17:31:33 +0100
Subject: [PATCH 083/111] added extra test for conversion to yamlObject

---
 tested/nat_translation.py    |  3 ++-
 tests/test_preprocess_dsl.py | 25 ++++++++++++++++++++++---
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 117769ae..222a5922 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -120,7 +120,8 @@ def to_yaml_object(data: Any) -> YamlObject:
                 return ReturnOracle(result)
             if data["__tag__"] == "!expression":
                 return ExpressionString(to_yaml_object(value))
-            return value
+            return to_yaml_object(value)
+        return {k: to_yaml_object(v) for k, v in data.items()}
     elif isinstance(data, list):
         return [to_yaml_object(value) for value in data]
 
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 8e5b4783..8d12a571 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -4,14 +4,15 @@
 from pytest_mock import MockerFixture
 
 import tested
-from tested.dsl.translate_parser import _parse_yaml, _validate_dsl
+from tested.dsl.translate_parser import _parse_yaml, _validate_dsl, ExpressionString, \
+    ReturnOracle
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
     parse_yaml,
     run_translation,
     translate_yaml,
-    validate_pre_dsl,
+    validate_pre_dsl, to_yaml_object,
 )
 
 test_unit_yaml_str = """
@@ -227,7 +228,6 @@ def test_natural_translate_unit_test():
 """.strip()
     environment = create_enviroment()
     parsed_yaml = parse_yaml(test_unit_yaml_str)
-    print(parsed_yaml)
     translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
     translated_yaml = convert_to_yaml(translated_dsl)
     assert translated_yaml.strip() == translated_yaml_str
@@ -435,6 +435,25 @@ def test_parsing_failed():
     else:
         assert False, "Expected MarkedYAMLError error"
 
+def test_to_yaml_object():
+
+    environment = create_enviroment()
+    parsed_yaml = parse_yaml(test_unit_yaml_str)
+    translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
+    yaml_object = to_yaml_object(translated_dsl)
+    assert isinstance(yaml_object, dict)
+    tabs = yaml_object["tabs"]
+    assert isinstance(tabs, list)
+    context = tabs[0]["contexts"][0]
+    assert isinstance(context, dict)
+    testcase = context["testcases"][2]
+    assert isinstance(testcase, dict)
+    assert isinstance(testcase["return"], ExpressionString)
+
+    testcase = context["testcases"][3]
+    assert isinstance(testcase, dict)
+    assert isinstance(testcase["return"], ReturnOracle)
+
 
 def test_run_is_correct_when_no_file():
 

From 9ff38b5dcd35598586e29cf59675b8708f3b6238 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 18:30:45 +0100
Subject: [PATCH 084/111] made an extra test

---
 tested/nat_translation.py    |  2 +-
 tests/test_preprocess_dsl.py | 52 ++++++++++++++++++++++++++++++++----
 2 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 222a5922..5904d549 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -142,7 +142,7 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
     file_name = yaml_path.name
     split_name = file_name.split(".")
     path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
-    with open(path_to_new_yaml, "w") as yaml_file:
+    with open(path_to_new_yaml, "w", encoding="utf-8") as yaml_file:
         yaml_file.write(yaml_string)
 
 
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 8d12a571..a768d15c 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -4,15 +4,20 @@
 from pytest_mock import MockerFixture
 
 import tested
-from tested.dsl.translate_parser import _parse_yaml, _validate_dsl, ExpressionString, \
-    ReturnOracle
+from tested.dsl.translate_parser import (
+    ExpressionString,
+    ReturnOracle,
+    _parse_yaml,
+    _validate_dsl,
+)
 from tested.nat_translation import (
     convert_to_yaml,
     create_enviroment,
     parse_yaml,
     run_translation,
+    to_yaml_object,
     translate_yaml,
-    validate_pre_dsl, to_yaml_object,
+    validate_pre_dsl,
 )
 
 test_unit_yaml_str = """
@@ -361,7 +366,7 @@ def test_wrong_natural_translation_suite():
 - tab: animals
   testcases:
   - expression: tests(11)
-    return: 11
+    return: !exp 11
   - expression:
       javascript: animals_javascript(1 + 1)
       typescript: animals_typescript(1 + 1)
@@ -418,6 +423,43 @@ def test_run_is_correct(mocker: MockerFixture):
     }
 
 
+def test_file_is_generated(mocker: MockerFixture):
+    s = mocker.spy(
+        tested.nat_translation, name="generate_new_yaml"
+    )  # type: ignore[reportAttributeAccessIssue]
+
+    mock_files = [
+        mocker.mock_open(read_data=content).return_value
+        for content in [
+            """
+tabs:
+- tab: task3
+  testcases:
+  - statement: !natural_language
+        nl: resultaten = Proberen(10)
+        en: results = Tries(10)
+  - expression: !natural_language
+        nl: tel_woorden(resultaten)
+        en: count_words(results)
+    return: !natural_language
+        nl: Het resultaat is 10
+        en: The result is 10"""
+        ]
+    ]
+    mock_files.append(mocker.mock_open(read_data="{}").return_value)
+    mock_files.append(mocker.mock_open().return_value)
+    mock_opener = mocker.mock_open()
+    mock_opener.side_effect = mock_files
+    mocker.patch("builtins.open", mock_opener)
+
+    run_translation(Path("suite.yaml"), "en", True)
+
+    assert s.call_count == 1
+
+    # Check if the file was opened for writing
+    mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8")
+
+
 def test_parsing_failed():
     yaml_str = """
 tabs:
@@ -435,6 +477,7 @@ def test_parsing_failed():
     else:
         assert False, "Expected MarkedYAMLError error"
 
+
 def test_to_yaml_object():
 
     environment = create_enviroment()
@@ -456,7 +499,6 @@ def test_to_yaml_object():
 
 
 def test_run_is_correct_when_no_file():
-
     try:
         run_translation(Path("suite.yaml"), "en", False)
     except FileNotFoundError:

From 361aa5cf12f2cd140369f990884b51ed84b0de1b Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 19:28:14 +0100
Subject: [PATCH 085/111] made one more test

---
 tests/test_preprocess_dsl.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index a768d15c..1e7915b0 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -425,8 +425,8 @@ def test_run_is_correct(mocker: MockerFixture):
 
 def test_file_is_generated(mocker: MockerFixture):
     s = mocker.spy(
-        tested.nat_translation, name="generate_new_yaml"
-    )  # type: ignore[reportAttributeAccessIssue]
+        tested.nat_translation, name="generate_new_yaml" # type: ignore[reportAttributeAccessIssue]
+    )
 
     mock_files = [
         mocker.mock_open(read_data=content).return_value
@@ -497,6 +497,31 @@ def test_to_yaml_object():
     assert isinstance(testcase, dict)
     assert isinstance(testcase["return"], ReturnOracle)
 
+def test_dumper():
+    translated_dsl = {"__tag__": '!tag', "value": [1, 2, 3]}
+    yaml_str = convert_to_yaml(translated_dsl).strip()
+    expected = """
+!tag
+- 1
+- 2
+- 3
+    """.strip()
+    assert yaml_str == expected
+
+    translated_dsl = {"__tag__": '!tag', "value": {"key1": "value1", "key2": "value2"}}
+    yaml_str = convert_to_yaml(translated_dsl).strip()
+    expected = """
+!tag
+key1: value1
+key2: value2
+        """.strip()
+    assert yaml_str == expected
+
+    translated_dsl = {"__tag__": '!tag', "value": "value"}
+    yaml_str = convert_to_yaml(translated_dsl).strip()
+    expected = "!tag 'value'"
+    assert yaml_str == expected
+
 
 def test_run_is_correct_when_no_file():
     try:

From b81e152663b307678cf7fdd88d3e43647de6b017 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 21 Mar 2025 19:56:42 +0100
Subject: [PATCH 086/111] fix linting

---
 tests/test_preprocess_dsl.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 1e7915b0..1e2dc10b 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -425,7 +425,7 @@ def test_run_is_correct(mocker: MockerFixture):
 
 def test_file_is_generated(mocker: MockerFixture):
     s = mocker.spy(
-        tested.nat_translation, name="generate_new_yaml" # type: ignore[reportAttributeAccessIssue]
+        tested.nat_translation, name="generate_new_yaml"  # type: ignore[reportAttributeAccessIssue]
     )
 
     mock_files = [
@@ -497,8 +497,9 @@ def test_to_yaml_object():
     assert isinstance(testcase, dict)
     assert isinstance(testcase["return"], ReturnOracle)
 
+
 def test_dumper():
-    translated_dsl = {"__tag__": '!tag', "value": [1, 2, 3]}
+    translated_dsl = {"__tag__": "!tag", "value": [1, 2, 3]}
     yaml_str = convert_to_yaml(translated_dsl).strip()
     expected = """
 !tag
@@ -508,7 +509,7 @@ def test_dumper():
     """.strip()
     assert yaml_str == expected
 
-    translated_dsl = {"__tag__": '!tag', "value": {"key1": "value1", "key2": "value2"}}
+    translated_dsl = {"__tag__": "!tag", "value": {"key1": "value1", "key2": "value2"}}
     yaml_str = convert_to_yaml(translated_dsl).strip()
     expected = """
 !tag
@@ -517,7 +518,7 @@ def test_dumper():
         """.strip()
     assert yaml_str == expected
 
-    translated_dsl = {"__tag__": '!tag', "value": "value"}
+    translated_dsl = {"__tag__": "!tag", "value": "value"}
     yaml_str = convert_to_yaml(translated_dsl).strip()
     expected = "!tag 'value'"
     assert yaml_str == expected

From fc7258eb81706f82f4796eecede418e4399e759a Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 25 Mar 2025 20:01:38 +0100
Subject: [PATCH 087/111] Major cutback of of TESTed in translation

---
 .devcontainer/dodona-tested.dockerfile |  3 +-
 tested/dsl/translate_parser.py         | 78 +-------------------------
 tested/main.py                         |  4 +-
 tested/nat_translation.py              | 54 +++++++++---------
 4 files changed, 35 insertions(+), 104 deletions(-)

diff --git a/.devcontainer/dodona-tested.dockerfile b/.devcontainer/dodona-tested.dockerfile
index d7172452..95a87646 100644
--- a/.devcontainer/dodona-tested.dockerfile
+++ b/.devcontainer/dodona-tested.dockerfile
@@ -43,7 +43,8 @@ RUN <<EOF
         pyyaml==6.0.1 \
         Pygments==2.18.0 \
         python-i18n==0.3.9 \
-        pylint==3.0.1
+        pylint==3.0.1 \
+        pystache==0.6.8
 
     # C/C++ dependencies
     apt-get install -y --no-install-recommends \
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index f1298fd5..403c2e31 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -33,6 +33,7 @@
 )
 from tested.dodona import ExtendedMessage
 from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string
+from tested.dsl.dsl_errors import print_dsl_validation_errors, raise_yaml_error
 from tested.parsing import get_converter, suite_to_json
 from tested.serialisation import (
     BooleanType,
@@ -138,34 +139,6 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     return ReturnOracle(result)
 
 
-def raise_yaml_error(yaml_stream: str, exc: yaml.MarkedYAMLError):
-    lines = yaml_stream.splitlines()
-
-    if exc.problem_mark is None:
-        # There is no additional information, so what can we do?
-        raise exc
-
-    sys.stderr.write(
-        textwrap.dedent(
-            f"""
-            YAML error while parsing test suite. This means there is a YAML syntax error.
-
-            The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}:
-
-                {lines[exc.problem_mark.line]}
-                {" " * exc.problem_mark.column + "^"}
-
-            The error message was:
-                {exc.problem} {exc.context}
-
-            The detailed exception is provided below.
-            You might also find help by validating your YAML file with a YAML validator.\n
-            """
-        )
-    )
-    raise exc
-
-
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
     Parse a string or stream to YAML.
@@ -221,14 +194,6 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
 _SCHEMA_VALIDATOR = load_schema_validator()
 
 
-class DslValidationError(ValueError):
-    pass
-
-
-class InvalidYamlError(ValueError):
-    pass
-
-
 @define(frozen=True)
 class DslContext:
     """
@@ -281,32 +246,6 @@ def merge_inheritable_with_specific_config(
         return recursive_dict_merge(inherited_options, specific_options)
 
 
-def convert_validation_error_to_group(
-    error: ValidationError,
-) -> ExceptionGroup | Exception:
-    if not error.context and not error.cause:
-        if len(error.message) > 150:
-            message = error.message.replace(str(error.instance), "<DSL>")
-            note = "With <DSL> being: " + textwrap.shorten(str(error.instance), 500)
-        else:
-            message = error.message
-            note = None
-        converted = DslValidationError(
-            f"Validation error at {error.json_path}: " + message
-        )
-        if note:
-            converted.add_note(note)
-        return converted
-    elif error.cause:
-        return error.cause
-    elif error.context:
-        causes = [convert_validation_error_to_group(x) for x in error.context]
-        message = f"Validation error at {error.json_path}, caused by a sub-exception."
-        return ExceptionGroup(message, causes)
-    else:
-        return error
-
-
 def _validate_dsl(dsl_object: YamlObject):
     """
     Validate a DSl object.
@@ -315,20 +254,7 @@ def _validate_dsl(dsl_object: YamlObject):
     :return: True if valid, False otherwise.
     """
     errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
-    if len(errors) == 1:
-        message = (
-            "Validating the DSL resulted in an error. "
-            "The most specific sub-exception is often the culprit. "
-        )
-        error = convert_validation_error_to_group(errors[0])
-        if isinstance(error, ExceptionGroup):
-            raise ExceptionGroup(message, error.exceptions)
-        else:
-            raise DslValidationError(message + str(error)) from error
-    elif len(errors) > 1:
-        the_errors = [convert_validation_error_to_group(e) for e in errors]
-        message = "Validating the DSL resulted in some errors."
-        raise ExceptionGroup(message, the_errors)
+    print_dsl_validation_errors(errors)
 
 
 def _tested_type_to_value(tested_type: TestedType) -> Value:
diff --git a/tested/main.py b/tested/main.py
index c7052208..c57d87e5 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -13,7 +13,7 @@
 from tested.testsuite import parse_test_suite
 
 
-def run(config: DodonaConfig, judge_output: IO, language: str = "-"):
+def run(config: DodonaConfig, judge_output: IO, language: str = None):
     """
     Run the TESTed judge.
 
@@ -21,7 +21,7 @@ def run(config: DodonaConfig, judge_output: IO, language: str = "-"):
     :param judge_output: Where the judge output will be written to.
     :param language: The language to use to translate the test-suite.
     """
-    if language == "-":
+    if language is None:
         try:
             with open(f"{config.resources}/{config.test_suite}", "r") as t:
                 textual_suite = t.read()
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 5904d549..baf912c5 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,46 +1,40 @@
+import json
 import os
+import re
 import sys
+# import pystache
 from pathlib import Path
-from typing import Any, cast
+from typing import Any, cast, Type
 
 import yaml
-from jinja2 import Environment
+from jinja2 import Environment, TemplateSyntaxError
+from jsonschema.protocols import Validator
+from jsonschema.validators import validator_for
 from yaml.nodes import MappingNode, ScalarNode, SequenceNode
 
+from tested.dsl.dsl_errors import print_dsl_validation_errors, raise_yaml_error
 from tested.dsl.translate_parser import (
-    DslValidationError,
     ExpressionString,
     ReturnOracle,
     YamlObject,
     _validate_dsl,
-    convert_validation_error_to_group,
-    load_schema_validator,
-    raise_yaml_error,
 )
 
 
-def validate_pre_dsl(dsl_object: YamlObject):
+def validate_pre_dsl(yaml_object: Any):
     """
     Validate a DSl object.
-    :param dsl_object: The object to validate.
+    :param yaml_object: The object to validate.
     :return: True if valid, False otherwise.
     """
-    _SCHEMA_VALIDATOR = load_schema_validator("schema-strict-nat-translation.json")
-    errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
-    if len(errors) == 1:
-        message = (
-            "Validating the DSL resulted in an error. "
-            "The most specific sub-exception is often the culprit. "
-        )
-        error = convert_validation_error_to_group(errors[0])
-        if isinstance(error, ExceptionGroup):
-            raise ExceptionGroup(message, error.exceptions)
-        else:
-            raise DslValidationError(message + str(error)) from error
-    elif len(errors) > 1:
-        the_errors = [convert_validation_error_to_group(e) for e in errors]
-        message = "Validating the DSL resulted in some errors."
-        raise ExceptionGroup(message, the_errors)
+    path_to_schema = Path(__file__).parent / "dsl/schema-strict-nat-translation.json"
+    with open(path_to_schema, "r") as schema_file:
+        schema_object = json.load(schema_file)
+
+    validator: Type[Validator] = validator_for(schema_object)
+    schema_validator = validator(schema_object)
+    errors = list(schema_validator.iter_errors(yaml_object))
+    print_dsl_validation_errors(errors)
 
 
 class CustomDumper(yaml.SafeDumper):
@@ -105,8 +99,18 @@ def translate_yaml(
         }
     elif isinstance(data, list):
         return [translate_yaml(item, translations, language, env) for item in data]
-    elif isinstance(data, str):
+    elif isinstance(data, str) and translations:
+        # return pystache.render(data, translations)
         return env.from_string(data).render(translations)
+        # try:
+        #     data = "{% raw %}" + data + "{% endraw %}"
+        #     return env.from_string(data).render(translations)
+        # except TemplateSyntaxError as e:
+        #     print(f"error: {e}")
+        #     print(f"data {data}")
+        #     return re.sub(r"{{\s*(.*?)\s*}}",
+        #                   lambda m: translations.get(m.group(1).strip(), m.group(0)),
+        #                   data)
     return data
 
 

From 18fbd0213eecce6da1fda6879622c16e24efdcb2 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 25 Mar 2025 21:06:16 +0100
Subject: [PATCH 088/111] forgot to push actual new file

---
 tested/dsl/dsl_errors.py     |  80 ++++++++++++++
 tests/test_preprocess_dsl.py | 199 ++++++++++++++++++++++++++++-------
 2 files changed, 242 insertions(+), 37 deletions(-)
 create mode 100644 tested/dsl/dsl_errors.py

diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py
new file mode 100644
index 00000000..08ab6101
--- /dev/null
+++ b/tested/dsl/dsl_errors.py
@@ -0,0 +1,80 @@
+import sys
+import textwrap
+
+import yaml
+from jsonschema.exceptions import ValidationError
+
+class InvalidYamlError(ValueError):
+    pass
+
+class DslValidationError(ValueError):
+    pass
+
+def convert_validation_error_to_group(
+    error: ValidationError,
+) -> ExceptionGroup | Exception:
+    if not error.context and not error.cause:
+        if len(error.message) > 150:
+            message = error.message.replace(str(error.instance), "<DSL>")
+            note = "With <DSL> being: " + textwrap.shorten(str(error.instance), 500)
+        else:
+            message = error.message
+            note = None
+        converted = DslValidationError(
+            f"Validation error at {error.json_path}: " + message
+        )
+        if note:
+            converted.add_note(note)
+        return converted
+    elif error.cause:
+        return error.cause
+    elif error.context:
+        causes = [convert_validation_error_to_group(x) for x in error.context]
+        message = f"Validation error at {error.json_path}, caused by a sub-exception."
+        return ExceptionGroup(message, causes)
+    else:
+        return error
+
+def print_dsl_validation_errors(errors: list):
+    if len(errors) == 1:
+        message = (
+            "Validating the DSL resulted in an error. "
+            "The most specific sub-exception is often the culprit. "
+        )
+        error = convert_validation_error_to_group(errors[0])
+        if isinstance(error, ExceptionGroup):
+            raise ExceptionGroup(message, error.exceptions)
+        else:
+            raise DslValidationError(message + str(error)) from error
+    elif len(errors) > 1:
+        the_errors = [convert_validation_error_to_group(e) for e in errors]
+        message = "Validating the DSL resulted in some errors."
+        raise ExceptionGroup(message, the_errors)
+
+
+def raise_yaml_error(yaml_stream: str, exc: yaml.MarkedYAMLError):
+    lines = yaml_stream.splitlines()
+
+    if exc.problem_mark is None:
+        # There is no additional information, so what can we do?
+        raise exc
+
+    sys.stderr.write(
+        textwrap.dedent(
+            f"""
+            YAML error while parsing test suite. This means there is a YAML syntax error.
+
+            The YAML parser indicates the problem lies at line {exc.problem_mark.line + 1}, column {exc.problem_mark.column + 1}:
+
+                {lines[exc.problem_mark.line]}
+                {" " * exc.problem_mark.column + "^"}
+
+            The error message was:
+                {exc.problem} {exc.context}
+
+            The detailed exception is provided below.
+            You might also find help by validating your YAML file with a YAML validator.\n
+            """
+        )
+    )
+    raise exc
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 1e2dc10b..94c396da 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -93,6 +93,75 @@
                 en: "Eleven_{{elf}}"
                 nl: "Elf_{{elf}}"
               format: "code"
+""".strip()
+
+def validate_natural_translate(yaml_str: str, translated_yaml_str: str):
+    enviroment = create_enviroment()
+    yaml_object = parse_yaml(yaml_str)
+    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
+    translated_yaml = convert_to_yaml(translated_dsl)
+    assert translated_yaml.strip() == translated_yaml_str
+
+def test_files_and_descriptions():
+    pass
+
+def test_return():
+    yaml_str = """
+translations:
+  animal:
+    en: "animals"
+    nl: "dieren"
+  result:
+    en: "results"
+    nl: "resultaten"
+tabs:
+  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}"
+    translations:
+      animal:
+        en: "animal_tab"
+        nl: "dier_tab"
+    contexts:
+      - testcases:
+        - expression: !natural_language
+            en: "count"
+            nl: "tellen"
+          return: !natural_language
+            en: !expression 'count'
+            nl: !expression 'tellen'
+        - expression: 'ok(10)'
+          return: !oracle
+            value: !natural_language
+              en: "The {{result}} 10 is OK!"
+              nl: "Het {{result}} 10 is OK!"
+            oracle: "custom_check"
+            file: "test.py"
+            name: "evaluate_test"
+            arguments: !natural_language
+              en: ["The value", "is OK!", "is not OK!"]
+              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
+    """.strip()
+    translated_yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python: animals_python_en(1 + 1)
+    return: 2
+""".strip()
+    validate_natural_translate(yaml_str, translated_yaml_str)
+
+def test_nat_lang_and_prog_lang_combo():
+    yaml_str = """
+translations:
+  animal:
+    en: "animals"
+    nl: "dieren"
+tabs:
   - tab: '{{animal}}'
     testcases:
       - expression: !natural_language
@@ -107,10 +176,50 @@
             en: "{{animal}}_python_en(1 + 1)"
             nl: "{{animal}}_python_nl(1 + 1)"
         return: 2
+""".strip()
+    translated_yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python: animals_python_en(1 + 1)
+    return: 2
+""".strip()
+    validate_natural_translate(yaml_str, translated_yaml_str)
+
+def test_format_expression():
+    yaml_str = """
+translations:
+  select:
+    en: "select"
+    nl: "selecteer"
+tabs:
   - tab: 'test'
     testcases:
       - expression: "{{select}}('a', {'a': 1, 'b': 2})"
         return: 1
+""".strip()
+    translated_yaml_str = """
+tabs:
+- tab: test
+  testcases:
+  - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
+    return: 1
+""".strip()
+    validate_natural_translate(yaml_str, translated_yaml_str)
+
+def test_natural_translate_context():
+    yaml_str = """
+translations:
+  result:
+    en: "results"
+    nl: "resultaten"
+tabs:
   - tab: "task"
     contexts: !natural_language
         en:
@@ -127,6 +236,27 @@
               return: 'Het {{result}} is 10'
             - expression: "tellen"
               return: !expression 'tellen'
+""".strip()
+    translated_yaml_str = """
+tabs:
+- tab: task
+  contexts:
+  - testcases:
+    - statement: results = Trying(10)
+    - expression: count_words(results)
+      return: The results is 10
+    - expression: count
+      return: !expression 'count'
+""".strip()
+    validate_natural_translate(yaml_str, translated_yaml_str)
+
+def test_natural_translate_testcases_in_context():
+    yaml_str = """
+translations:
+  result:
+    en: "results"
+    nl: "resultaten"
+tabs:
   - tab: "task2"
     contexts:
         - testcases: !natural_language
@@ -142,6 +272,27 @@
                 return: 'Het {{result}} is 10'
               - expression: "tellen"
                 return: !expression 'tellen'
+""".strip()
+    translated_yaml_str = """
+tabs:
+- tab: task2
+  contexts:
+  - testcases:
+    - statement: results = Trying(10)
+    - expression: count_words(results)
+      return: The results is 10
+    - expression: count
+      return: !expression 'count'
+""".strip()
+    validate_natural_translate(yaml_str, translated_yaml_str)
+
+def test_natural_translate_testcases():
+    yaml_str = """
+translations:
+  result:
+    en: "results"
+    nl: "resultaten"
+tabs:
   - tab: "task3"
     testcases: !natural_language
         en:
@@ -157,6 +308,17 @@
           - expression: "tellen"
             return: !expression 'tellen'
 """.strip()
+    translated_yaml_str = """
+tabs:
+- tab: task3
+  testcases:
+  - statement: results = Trying(10)
+  - expression: count_words(results)
+    return: The results is 10
+  - expression: count
+    return: !expression 'count'
+""".strip()
+    validate_natural_translate(yaml_str, translated_yaml_str)
 
 
 def test_natural_translate_unit_test():
@@ -193,43 +355,6 @@ def test_natural_translate_unit_test():
       description:
         description: Eleven_eleven
         format: code
-- tab: animals
-  testcases:
-  - expression: tests(11)
-    return: 11
-  - expression:
-      javascript: animals_javascript(1 + 1)
-      typescript: animals_typescript(1 + 1)
-      java: Submission.animals_java(1 + 1)
-      python: animals_python_en(1 + 1)
-    return: 2
-- tab: test
-  testcases:
-  - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
-    return: 1
-- tab: task
-  contexts:
-  - testcases:
-    - statement: results = Trying(10)
-    - expression: count_words(results)
-      return: The results is 10
-    - expression: count
-      return: !expression 'count'
-- tab: task2
-  contexts:
-  - testcases:
-    - statement: results = Trying(10)
-    - expression: count_words(results)
-      return: The results is 10
-    - expression: count
-      return: !expression 'count'
-- tab: task3
-  testcases:
-  - statement: results = Trying(10)
-  - expression: count_words(results)
-    return: The results is 10
-  - expression: count
-    return: !expression 'count'
 """.strip()
     environment = create_enviroment()
     parsed_yaml = parse_yaml(test_unit_yaml_str)

From e86d48a0d91f9266bf7995f9ddcdac125ec376ff Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 26 Mar 2025 15:03:40 +0100
Subject: [PATCH 089/111] split up tests

---
 tested/dsl/dsl_errors.py       |   2 +-
 tested/dsl/translate_parser.py |   4 +-
 tested/nat_translation.py      |   4 +-
 tests/test_preprocess_dsl.py   | 297 +++++++++++++--------------------
 4 files changed, 124 insertions(+), 183 deletions(-)

diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py
index 08ab6101..fc4a3917 100644
--- a/tested/dsl/dsl_errors.py
+++ b/tested/dsl/dsl_errors.py
@@ -35,7 +35,7 @@ def convert_validation_error_to_group(
     else:
         return error
 
-def print_dsl_validation_errors(errors: list):
+def handle_dsl_validation_errors(errors: list):
     if len(errors) == 1:
         message = (
             "Validating the DSL resulted in an error. "
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 403c2e31..499a11c1 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -33,7 +33,7 @@
 )
 from tested.dodona import ExtendedMessage
 from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string
-from tested.dsl.dsl_errors import print_dsl_validation_errors, raise_yaml_error
+from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error
 from tested.parsing import get_converter, suite_to_json
 from tested.serialisation import (
     BooleanType,
@@ -254,7 +254,7 @@ def _validate_dsl(dsl_object: YamlObject):
     :return: True if valid, False otherwise.
     """
     errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
-    print_dsl_validation_errors(errors)
+    handle_dsl_validation_errors(errors)
 
 
 def _tested_type_to_value(tested_type: TestedType) -> Value:
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index baf912c5..eeb71ca9 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -12,7 +12,7 @@
 from jsonschema.validators import validator_for
 from yaml.nodes import MappingNode, ScalarNode, SequenceNode
 
-from tested.dsl.dsl_errors import print_dsl_validation_errors, raise_yaml_error
+from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error
 from tested.dsl.translate_parser import (
     ExpressionString,
     ReturnOracle,
@@ -34,7 +34,7 @@ def validate_pre_dsl(yaml_object: Any):
     validator: Type[Validator] = validator_for(schema_object)
     schema_validator = validator(schema_object)
     errors = list(schema_validator.iter_errors(yaml_object))
-    print_dsl_validation_errors(errors)
+    handle_dsl_validation_errors(errors)
 
 
 class CustomDumper(yaml.SafeDumper):
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 94c396da..1861630e 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -20,7 +20,16 @@
     validate_pre_dsl,
 )
 
-test_unit_yaml_str = """
+def validate_natural_translate(yaml_str: str, translated_yaml_str: str):
+    enviroment = create_enviroment()
+    yaml_object = parse_yaml(yaml_str)
+    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml.strip() == translated_yaml_str
+
+def test_files_and_descriptions():
+    yaml_str = """
 translations:
   animal:
     en: "animals"
@@ -28,46 +37,19 @@
   result:
     en: "results"
     nl: "resultaten"
-  elf:
-    en: "eleven"
-    nl: "elf"
-  select:
-    en: "select"
-    nl: "selecteer"
 tabs:
-  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}"
-    translations:
-      animal:
-        en: "animal_tab"
-        nl: "dier_tab"
+  - tab: "{{animal}}"
     contexts:
       - testcases:
         - statement: !natural_language
-            en: '{{result}} = Trying(10)'
-            nl: '{{result}} = Proberen(10)'
+            en: '{{result}}: dict = Trying("file.txt")'
+            nl: '{{result}}: dict = Proberen("fileNL.txt")'
         - expression: !natural_language
             en: 'count_words({{result}})'
             nl: 'tel_woorden({{result}})'
           return: !natural_language
             en: 'The {{result}} is 10'
             nl: 'Het {{result}} is 10'
-        - expression: !natural_language
-            en: "count"
-            nl: "tellen"
-          return: !natural_language
-            en: !expression 'count'
-            nl: !expression 'tellen'
-        - expression: 'ok(10)'
-          return: !oracle
-            value: !natural_language
-              en: "The {{result}} 10 is OK!"
-              nl: "Het {{result}} 10 is OK!"
-            oracle: "custom_check"
-            file: "test.py"
-            name: "evaluate_test"
-            arguments: !natural_language
-              en: ["The value", "is OK!", "is not OK!"]
-              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
           description: !natural_language
             en: "Ten"
             nl: "Tien"
@@ -82,28 +64,21 @@
           result:
             en: "results_context"
             nl: "resultaten_context"
-      - testcases:
-          - statement: !natural_language
-              en: 'result = Trying(11)'
-              nl: 'resultaat = Proberen(11)'
-          - expression: 'result'
-            return: '11_{elf}'
-            description:
-              description: !natural_language
-                en: "Eleven_{{elf}}"
-                nl: "Elf_{{elf}}"
-              format: "code"
+    """.strip()
+    translated_yaml_str = """
+tabs:
+- tab: animals
+  contexts:
+  - testcases:
+    - statement: 'results_context: dict = Trying("file.txt")'
+    - expression: count_words(results_context)
+      return: The results_context is 10
+      description: Ten
+    files:
+    - name: file.txt
+      url: media/workdir/file.txt
 """.strip()
-
-def validate_natural_translate(yaml_str: str, translated_yaml_str: str):
-    enviroment = create_enviroment()
-    yaml_object = parse_yaml(yaml_str)
-    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
-    translated_yaml = convert_to_yaml(translated_dsl)
-    assert translated_yaml.strip() == translated_yaml_str
-
-def test_files_and_descriptions():
-    pass
+    validate_natural_translate(yaml_str, translated_yaml_str)
 
 def test_return():
     yaml_str = """
@@ -142,16 +117,21 @@ def test_return():
     """.strip()
     translated_yaml_str = """
 tabs:
-- tab: animals
-  testcases:
-  - expression: tests(11)
-    return: 11
-  - expression:
-      javascript: animals_javascript(1 + 1)
-      typescript: animals_typescript(1 + 1)
-      java: Submission.animals_java(1 + 1)
-      python: animals_python_en(1 + 1)
-    return: 2
+- tab: '{animal_tab}_{results}'
+  contexts:
+  - testcases:
+    - expression: count
+      return: !expression 'count'
+    - expression: ok(10)
+      return: !oracle
+        value: The results 10 is OK!
+        oracle: custom_check
+        file: test.py
+        name: evaluate_test
+        arguments:
+        - The value
+        - is OK!
+        - is not OK!
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
@@ -320,52 +300,7 @@ def test_natural_translate_testcases():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
-
-def test_natural_translate_unit_test():
-    # Everywhere where !natural_language is used, it is mandatory to do so.
-    # Everywhere else it isn't.
-    translated_yaml_str = """
-tabs:
-- tab: '{animal_tab}_{results}'
-  contexts:
-  - testcases:
-    - statement: results_context = Trying(10)
-    - expression: count_words(results_context)
-      return: The results_context is 10
-    - expression: count
-      return: !expression 'count'
-    - expression: ok(10)
-      return: !oracle
-        value: The results_context 10 is OK!
-        oracle: custom_check
-        file: test.py
-        name: evaluate_test
-        arguments:
-        - The value
-        - is OK!
-        - is not OK!
-      description: Ten
-    files:
-    - name: file.txt
-      url: media/workdir/file.txt
-  - testcases:
-    - statement: result = Trying(11)
-    - expression: result
-      return: 11_{elf}
-      description:
-        description: Eleven_eleven
-        format: code
-""".strip()
-    environment = create_enviroment()
-    parsed_yaml = parse_yaml(test_unit_yaml_str)
-    translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
-    translated_yaml = convert_to_yaml(translated_dsl)
-    assert translated_yaml.strip() == translated_yaml_str
-
-
 def test_natural_translate_io_test():
-    # Everywhere where !natural_language is used, it is mandatory to do so.
-    # Everywhere else it isn't.
     yaml_str = """
 units:
   - unit: !natural_language
@@ -392,44 +327,6 @@ def test_natural_translate_io_test():
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
-        - stdin: !natural_language
-            en: "Friend of {{User}}"
-            nl: "Vriend van {{User}}"
-          arguments: !natural_language
-            en: [ "input", "output" ]
-            nl: [ "invoer", "uitvoer" ]
-          stdout:
-            data: !natural_language
-              en: "Hi Friend of {{User}}"
-              nl: "Hallo Vriend van {{User}}"
-            config:
-              ignoreWhitespace: true
-          stderr:
-            data: !natural_language
-              en: "Nothing to see here {{User}}"
-              nl: "Hier is niets te zien {{User}}"
-            config:
-              ignoreWhitespace: true
-          exception:
-            message: !natural_language
-              en: "Does not look good {{User}}"
-              nl: "Ziet er niet goed uit {{User}}"
-            types:
-              typescript: "ERROR"
-  - unit: "test"
-    scripts:
-      - expression: !natural_language
-            en: "tests(11)"
-            nl: "testen(11)"
-        return: 11
-  - unit: "test2"
-    scripts: !natural_language
-      en: 
-        - expression: "tests(11)"
-          return: 11
-      nl:
-        - expression: "testen(11)"
-          return: 11
 """.strip()
     translated_yaml_str = """
 units:
@@ -443,40 +340,48 @@ def test_natural_translate_io_test():
       stdout: Hi user
       stderr: Nothing to see here user
       exception: Does not look good
-    - stdin: Friend of user
-      arguments:
-      - input
-      - output
-      stdout:
-        data: Hi Friend of user
-        config:
-          ignoreWhitespace: true
-      stderr:
-        data: Nothing to see here user
-        config:
-          ignoreWhitespace: true
-      exception:
-        message: Does not look good user
-        types:
-          typescript: ERROR
-- unit: test
-  scripts:
-  - expression: tests(11)
-    return: 11
-- unit: test2
-  scripts:
-  - expression: tests(11)
-    return: 11
 """.strip()
-    enviroment = create_enviroment()
-    yaml_object = parse_yaml(yaml_str)
-    translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
-    translated_yaml = convert_to_yaml(translated_dsl)
-    assert translated_yaml.strip() == translated_yaml_str
+    validate_natural_translate(yaml_str, translated_yaml_str)
 
 
 def test_validation():
-    yaml_object = parse_yaml(test_unit_yaml_str)
+    yaml_str = """
+translations:
+  animal:
+    en: "animals"
+    nl: "dieren"
+  result:
+    en: "results"
+    nl: "resultaten"
+  elf:
+    en: "eleven"
+    nl: "elf"
+  select:
+    en: "select"
+    nl: "selecteer"
+tabs:
+  - tab: "{{animal}}"
+    contexts:
+      - testcases:
+        - expression: !natural_language
+            en: "count"
+            nl: "tellen"
+          return: !natural_language
+            en: !expression 'count'
+            nl: !expression 'tellen'
+        - expression: 'ok(10)'
+          return: !oracle
+            value: !natural_language
+              en: "The {{result}} 10 is OK!"
+              nl: "Het {{result}} 10 is OK!"
+            oracle: "custom_check"
+            file: "test.py"
+            name: "evaluate_test"
+            arguments: !natural_language
+              en: ["The value", "is OK!", "is not OK!"]
+              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
+        """
+    yaml_object = parse_yaml(yaml_str)
     validate_pre_dsl(yaml_object)
 
     enviroment = create_enviroment()
@@ -604,9 +509,45 @@ def test_parsing_failed():
 
 
 def test_to_yaml_object():
+    yaml_str = """
+translations:
+  animal:
+    en: "animals"
+    nl: "dieren"
+  result:
+    en: "results"
+    nl: "resultaten"
+  elf:
+    en: "eleven"
+    nl: "elf"
+  select:
+    en: "select"
+    nl: "selecteer"
+tabs:
+  - tab: "{{animal}}"
+    contexts:
+      - testcases:
+        - expression: !natural_language
+            en: "count"
+            nl: "tellen"
+          return: !natural_language
+            en: !expression 'count'
+            nl: !expression 'tellen'
+        - expression: 'ok(10)'
+          return: !oracle
+            value: !natural_language
+              en: "The {{result}} 10 is OK!"
+              nl: "Het {{result}} 10 is OK!"
+            oracle: "custom_check"
+            file: "test.py"
+            name: "evaluate_test"
+            arguments: !natural_language
+              en: ["The value", "is OK!", "is not OK!"]
+              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
+    """
 
     environment = create_enviroment()
-    parsed_yaml = parse_yaml(test_unit_yaml_str)
+    parsed_yaml = parse_yaml(yaml_str)
     translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
     yaml_object = to_yaml_object(translated_dsl)
     assert isinstance(yaml_object, dict)
@@ -614,11 +555,11 @@ def test_to_yaml_object():
     assert isinstance(tabs, list)
     context = tabs[0]["contexts"][0]
     assert isinstance(context, dict)
-    testcase = context["testcases"][2]
+    testcase = context["testcases"][0]
     assert isinstance(testcase, dict)
     assert isinstance(testcase["return"], ExpressionString)
 
-    testcase = context["testcases"][3]
+    testcase = context["testcases"][1]
     assert isinstance(testcase, dict)
     assert isinstance(testcase["return"], ReturnOracle)
 

From 428991080b0583c48021b6df8af27fbc839bafd2 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 26 Mar 2025 15:06:03 +0100
Subject: [PATCH 090/111] Fixed linting and typing issues

---
 tested/dsl/dsl_errors.py     | 4 ++++
 tested/main.py               | 2 +-
 tested/nat_translation.py    | 3 ++-
 tests/test_preprocess_dsl.py | 9 +++++++++
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py
index fc4a3917..6f1c4d93 100644
--- a/tested/dsl/dsl_errors.py
+++ b/tested/dsl/dsl_errors.py
@@ -4,12 +4,15 @@
 import yaml
 from jsonschema.exceptions import ValidationError
 
+
 class InvalidYamlError(ValueError):
     pass
 
+
 class DslValidationError(ValueError):
     pass
 
+
 def convert_validation_error_to_group(
     error: ValidationError,
 ) -> ExceptionGroup | Exception:
@@ -35,6 +38,7 @@ def convert_validation_error_to_group(
     else:
         return error
 
+
 def handle_dsl_validation_errors(errors: list):
     if len(errors) == 1:
         message = (
diff --git a/tested/main.py b/tested/main.py
index c57d87e5..1b78bf3c 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -13,7 +13,7 @@
 from tested.testsuite import parse_test_suite
 
 
-def run(config: DodonaConfig, judge_output: IO, language: str = None):
+def run(config: DodonaConfig, judge_output: IO, language: str|None = None):
     """
     Run the TESTed judge.
 
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index eeb71ca9..c8187a1e 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,9 +2,10 @@
 import os
 import re
 import sys
+
 # import pystache
 from pathlib import Path
-from typing import Any, cast, Type
+from typing import Any, Type, cast
 
 import yaml
 from jinja2 import Environment, TemplateSyntaxError
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 1861630e..15ab2137 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -20,6 +20,7 @@
     validate_pre_dsl,
 )
 
+
 def validate_natural_translate(yaml_str: str, translated_yaml_str: str):
     enviroment = create_enviroment()
     yaml_object = parse_yaml(yaml_str)
@@ -28,6 +29,7 @@ def validate_natural_translate(yaml_str: str, translated_yaml_str: str):
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
+
 def test_files_and_descriptions():
     yaml_str = """
 translations:
@@ -80,6 +82,7 @@ def test_files_and_descriptions():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_return():
     yaml_str = """
 translations:
@@ -135,6 +138,7 @@ def test_return():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_nat_lang_and_prog_lang_combo():
     yaml_str = """
 translations:
@@ -172,6 +176,7 @@ def test_nat_lang_and_prog_lang_combo():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_format_expression():
     yaml_str = """
 translations:
@@ -193,6 +198,7 @@ def test_format_expression():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_natural_translate_context():
     yaml_str = """
 translations:
@@ -230,6 +236,7 @@ def test_natural_translate_context():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_natural_translate_testcases_in_context():
     yaml_str = """
 translations:
@@ -266,6 +273,7 @@ def test_natural_translate_testcases_in_context():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_natural_translate_testcases():
     yaml_str = """
 translations:
@@ -300,6 +308,7 @@ def test_natural_translate_testcases():
 """.strip()
     validate_natural_translate(yaml_str, translated_yaml_str)
 
+
 def test_natural_translate_io_test():
     yaml_str = """
 units:

From 33ba90f99c3c7d91c4fe6cf1f709a4aba78867ea Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 26 Mar 2025 15:11:50 +0100
Subject: [PATCH 091/111] Fixed ALL linting issues

---
 tested/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tested/main.py b/tested/main.py
index 1b78bf3c..e3148eac 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -13,7 +13,7 @@
 from tested.testsuite import parse_test_suite
 
 
-def run(config: DodonaConfig, judge_output: IO, language: str|None = None):
+def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
     """
     Run the TESTed judge.
 

From 204f50ad9013ac04b28250c74d7c981c91439f8f Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 27 Mar 2025 21:11:38 +0100
Subject: [PATCH 092/111] applied all changes regarding jinja2

---
 tested/configs.py            | 11 ++++++++-
 tested/judge/core.py         | 11 +++++++++
 tested/main.py               |  7 ++++--
 tested/nat_translation.py    | 46 +++++++++++++++++++++---------------
 tests/test_preprocess_dsl.py |  2 +-
 5 files changed, 54 insertions(+), 23 deletions(-)

diff --git a/tested/configs.py b/tested/configs.py
index 0cb0cc03..765e35e7 100644
--- a/tested/configs.py
+++ b/tested/configs.py
@@ -129,6 +129,7 @@ class Bundle:
     language: "Language"
     global_config: GlobalConfig
     out: IO
+    translations_missing_key: bool = False
 
     @property
     def config(self) -> DodonaConfig:
@@ -207,6 +208,7 @@ def create_bundle(
     output: IO,
     suite: Suite,
     language: str | None = None,
+    translations_missing_key: bool = False,
 ) -> Bundle:
     """
     Create a configuration bundle.
@@ -216,6 +218,8 @@ def create_bundle(
     :param suite: The test suite.
     :param language: Optional programming language. If None, the one from the Dodona
                      configuration will be used.
+    :param translations_missing_key: Indicator that the natural language translator
+    for the DSL key that was not defined in any translations map.
 
     :return: The configuration bundle.
     """
@@ -232,4 +236,9 @@ def create_bundle(
         suite=suite,
     )
     lang_config = langs.get_language(global_config, language)
-    return Bundle(language=lang_config, global_config=global_config, out=output)
+    return Bundle(
+        language=lang_config,
+        global_config=global_config,
+        out=output,
+        translations_missing_key=translations_missing_key,
+    )
diff --git a/tested/judge/core.py b/tested/judge/core.py
index 24bdd3b3..d5e7b8c4 100644
--- a/tested/judge/core.py
+++ b/tested/judge/core.py
@@ -10,7 +10,9 @@
     CloseContext,
     CloseJudgement,
     CloseTab,
+    ExtendedMessage,
     Metadata,
+    Permission,
     StartContext,
     StartJudgement,
     StartTab,
@@ -116,6 +118,15 @@ def judge(bundle: Bundle):
     # Do the set-up for the judgement.
     collector = OutputManager(bundle.out)
     collector.add(StartJudgement())
+    if bundle.translations_missing_key:
+        collector.add_messages(
+            [
+                ExtendedMessage(
+                    "The natural translator found a key that was not defined in the corresponding translations maps!",
+                    permission=Permission.STAFF,
+                )
+            ]
+        )
     max_time = float(bundle.config.time_limit) * 0.9
     start = time.perf_counter()
 
diff --git a/tested/main.py b/tested/main.py
index e3148eac..384575bc 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -40,14 +40,17 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
             suite = parse_dsl(textual_suite)
         else:
             suite = parse_test_suite(textual_suite)
+        missing_keys = False
     else:
-        translated_yaml_ob = run_translation(
+        translated_yaml_ob, missing_keys = run_translation(
             Path(f"{config.resources}/{config.test_suite}"),
             language=language,
             to_file=False,
         )
         suite = convert_dsl(translated_yaml_ob)
-    pack = create_bundle(config, judge_output, suite)
+    pack = create_bundle(
+        config, judge_output, suite, translations_missing_key=missing_keys
+    )
     from .judge import judge
 
     judge(pack)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index c8187a1e..07b1cd12 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,14 +1,11 @@
 import json
 import os
-import re
 import sys
-
-# import pystache
 from pathlib import Path
 from typing import Any, Type, cast
 
 import yaml
-from jinja2 import Environment, TemplateSyntaxError
+from jinja2 import Environment, TemplateSyntaxError, Undefined
 from jsonschema.protocols import Validator
 from jsonschema.validators import validator_for
 from yaml.nodes import MappingNode, ScalarNode, SequenceNode
@@ -101,17 +98,13 @@ def translate_yaml(
     elif isinstance(data, list):
         return [translate_yaml(item, translations, language, env) for item in data]
     elif isinstance(data, str) and translations:
-        # return pystache.render(data, translations)
-        return env.from_string(data).render(translations)
-        # try:
-        #     data = "{% raw %}" + data + "{% endraw %}"
-        #     return env.from_string(data).render(translations)
-        # except TemplateSyntaxError as e:
-        #     print(f"error: {e}")
-        #     print(f"data {data}")
-        #     return re.sub(r"{{\s*(.*?)\s*}}",
-        #                   lambda m: translations.get(m.group(1).strip(), m.group(0)),
-        #                   data)
+        try:
+            result = env.from_string(data).render(translations)
+            print("Missing keys:", TrackingUndefined.missing_keys)
+            print(result)
+            return result
+        except TemplateSyntaxError:
+            return data
     return data
 
 
@@ -137,8 +130,20 @@ def wrap_in_braces(value):
     return f"{{{value}}}"
 
 
+class TrackingUndefined(Undefined):
+    missing_keys = set()
+
+    def __str__(self):
+        # Store the missing key name
+        TrackingUndefined.missing_keys.add(self._undefined_name)
+        # Return it in Jinja syntax to keep it in the template
+        return f"{{{{ {self._undefined_name} }}}}"
+
+    __repr__ = __str__
+
+
 def create_enviroment() -> Environment:
-    enviroment = Environment()
+    enviroment = Environment(undefined=TrackingUndefined)
     enviroment.filters["braces"] = wrap_in_braces
     return enviroment
 
@@ -171,7 +176,9 @@ def parse_yaml(yaml_stream: str) -> Any:
         raise_yaml_error(yaml_stream, exc)
 
 
-def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObject:
+def run_translation(
+    path: Path, language: str, to_file: bool = True
+) -> tuple[YamlObject, bool]:
     try:
         with open(path, "r") as stream:
             yaml_stream = stream.read()
@@ -189,14 +196,15 @@ def run_translation(path: Path, language: str, to_file: bool = True) -> YamlObje
     enviroment = create_enviroment()
     translated_data = translate_yaml(parsed_yaml, {}, language, enviroment)
 
+    missing_keys = len(TrackingUndefined.missing_keys) > 0
     if to_file:
         translated_yaml_string = convert_to_yaml(translated_data)
         generate_new_yaml(path, translated_yaml_string, language)
-        return {}
+        return {}, missing_keys
     else:
         yaml_object = to_yaml_object(translated_data)
         _validate_dsl(yaml_object)
-        return yaml_object
+        return yaml_object, missing_keys
 
 
 if __name__ == "__main__":
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 15ab2137..f3fdc7ea 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -449,7 +449,7 @@ def test_run_is_correct(mocker: MockerFixture):
     mock_opener.side_effect = mock_files
     mocker.patch("builtins.open", mock_opener)
 
-    yaml_object = run_translation(Path("suite.yaml"), "en", False)
+    yaml_object, _ = run_translation(Path("suite.yaml"), "en", False)
 
     assert s.call_count == 0
     assert isinstance(yaml_object, dict)

From 9c83b4da9255427b771965f4aae469bade222b05 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 28 Mar 2025 12:11:26 +0100
Subject: [PATCH 093/111] Added an extra test

---
 .devcontainer/dodona-tested.dockerfile |  3 +--
 tested/nat_translation.py              |  3 ---
 tests/test_preprocess_dsl.py           | 22 ++++++++++++++++++++++
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/.devcontainer/dodona-tested.dockerfile b/.devcontainer/dodona-tested.dockerfile
index 95a87646..d7172452 100644
--- a/.devcontainer/dodona-tested.dockerfile
+++ b/.devcontainer/dodona-tested.dockerfile
@@ -43,8 +43,7 @@ RUN <<EOF
         pyyaml==6.0.1 \
         Pygments==2.18.0 \
         python-i18n==0.3.9 \
-        pylint==3.0.1 \
-        pystache==0.6.8
+        pylint==3.0.1
 
     # C/C++ dependencies
     apt-get install -y --no-install-recommends \
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 07b1cd12..c0228f08 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -23,7 +23,6 @@ def validate_pre_dsl(yaml_object: Any):
     """
     Validate a DSl object.
     :param yaml_object: The object to validate.
-    :return: True if valid, False otherwise.
     """
     path_to_schema = Path(__file__).parent / "dsl/schema-strict-nat-translation.json"
     with open(path_to_schema, "r") as schema_file:
@@ -100,8 +99,6 @@ def translate_yaml(
     elif isinstance(data, str) and translations:
         try:
             result = env.from_string(data).render(translations)
-            print("Missing keys:", TrackingUndefined.missing_keys)
-            print(result)
             return result
         except TemplateSyntaxError:
             return data
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index f3fdc7ea..5df58d6f 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -606,3 +606,25 @@ def test_run_is_correct_when_no_file():
         print("As expected")
     else:
         assert False, "Expected FileNotFoundError error"
+
+
+def test_template_syntax_error():
+    yaml_str = """
+translations:
+    works:
+        en: "works"
+        nl: "werkt"
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11{%{{works}}
+""".strip()
+    translated_yml = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11{%{{works}}
+""".strip()
+    validate_natural_translate(yaml_str, translated_yml)

From 29b3a45c3464effd94528d981c29f95b93c833b3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 31 Mar 2025 13:24:54 +0200
Subject: [PATCH 094/111] removed conversion to yamlObject

---
 tested/dsl/translate_parser.py |  4 ++--
 tested/main.py                 |  6 +++---
 tested/nat_translation.py      | 36 +++++-----------------------------
 tests/test_preprocess_dsl.py   |  7 ++++---
 4 files changed, 14 insertions(+), 39 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 499a11c1..a7b05504 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -650,7 +650,7 @@ def _convert_dsl_list(
     return objects
 
 
-def convert_dsl(dsl_object: YamlObject) -> Suite:
+def _convert_dsl(dsl_object: YamlObject) -> Suite:
     """
     Translate a DSL test suite into a full test suite.
 
@@ -692,7 +692,7 @@ def parse_dsl(dsl_string: str) -> Suite:
     """
     dsl_object = _parse_yaml(dsl_string)
     _validate_dsl(dsl_object)
-    return convert_dsl(dsl_object)
+    return _convert_dsl(dsl_object)
 
 
 def translate_to_test_suite(dsl_string: str) -> str:
diff --git a/tested/main.py b/tested/main.py
index 384575bc..096824e1 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -8,7 +8,7 @@
 
 from tested.configs import DodonaConfig, create_bundle
 from tested.dsl import parse_dsl
-from tested.dsl.translate_parser import convert_dsl
+from tested.dsl.translate_parser import _convert_dsl
 from tested.nat_translation import run_translation
 from tested.testsuite import parse_test_suite
 
@@ -42,12 +42,12 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
             suite = parse_test_suite(textual_suite)
         missing_keys = False
     else:
-        translated_yaml_ob, missing_keys = run_translation(
+        translated_yaml, missing_keys = run_translation(
             Path(f"{config.resources}/{config.test_suite}"),
             language=language,
             to_file=False,
         )
-        suite = convert_dsl(translated_yaml_ob)
+        suite = parse_dsl(translated_yaml)
     pack = create_bundle(
         config, judge_output, suite, translations_missing_key=missing_keys
     )
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index c0228f08..d409fd4e 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -11,12 +11,6 @@
 from yaml.nodes import MappingNode, ScalarNode, SequenceNode
 
 from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error
-from tested.dsl.translate_parser import (
-    ExpressionString,
-    ReturnOracle,
-    YamlObject,
-    _validate_dsl,
-)
 
 
 def validate_pre_dsl(yaml_object: Any):
@@ -96,7 +90,7 @@ def translate_yaml(
         }
     elif isinstance(data, list):
         return [translate_yaml(item, translations, language, env) for item in data]
-    elif isinstance(data, str) and translations:
+    elif isinstance(data, str):
         try:
             result = env.from_string(data).render(translations)
             return result
@@ -105,24 +99,6 @@ def translate_yaml(
     return data
 
 
-def to_yaml_object(data: Any) -> YamlObject:
-    if isinstance(data, dict):
-        if "__tag__" in data:
-            value = data["value"]
-            if data["__tag__"] == "!oracle":
-                result = to_yaml_object(value)
-                assert isinstance(result, dict)
-                return ReturnOracle(result)
-            if data["__tag__"] == "!expression":
-                return ExpressionString(to_yaml_object(value))
-            return to_yaml_object(value)
-        return {k: to_yaml_object(v) for k, v in data.items()}
-    elif isinstance(data, list):
-        return [to_yaml_object(value) for value in data]
-
-    return data
-
-
 def wrap_in_braces(value):
     return f"{{{value}}}"
 
@@ -175,7 +151,7 @@ def parse_yaml(yaml_stream: str) -> Any:
 
 def run_translation(
     path: Path, language: str, to_file: bool = True
-) -> tuple[YamlObject, bool]:
+) -> tuple[str, bool]:
     try:
         with open(path, "r") as stream:
             yaml_stream = stream.read()
@@ -194,14 +170,12 @@ def run_translation(
     translated_data = translate_yaml(parsed_yaml, {}, language, enviroment)
 
     missing_keys = len(TrackingUndefined.missing_keys) > 0
+    translated_yaml_string = convert_to_yaml(translated_data)
     if to_file:
-        translated_yaml_string = convert_to_yaml(translated_data)
         generate_new_yaml(path, translated_yaml_string, language)
-        return {}, missing_keys
+        return "", missing_keys
     else:
-        yaml_object = to_yaml_object(translated_data)
-        _validate_dsl(yaml_object)
-        return yaml_object, missing_keys
+        return translated_yaml_string, missing_keys
 
 
 if __name__ == "__main__":
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 5df58d6f..ba5f93bb 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -15,7 +15,6 @@
     create_enviroment,
     parse_yaml,
     run_translation,
-    to_yaml_object,
     translate_yaml,
     validate_pre_dsl,
 )
@@ -449,7 +448,8 @@ def test_run_is_correct(mocker: MockerFixture):
     mock_opener.side_effect = mock_files
     mocker.patch("builtins.open", mock_opener)
 
-    yaml_object, _ = run_translation(Path("suite.yaml"), "en", False)
+    translated_yaml, _ = run_translation(Path("suite.yaml"), "en", False)
+    yaml_object = parse_yaml(translated_yaml)
 
     assert s.call_count == 0
     assert isinstance(yaml_object, dict)
@@ -558,7 +558,8 @@ def test_to_yaml_object():
     environment = create_enviroment()
     parsed_yaml = parse_yaml(yaml_str)
     translated_dsl = translate_yaml(parsed_yaml, {}, "en", environment)
-    yaml_object = to_yaml_object(translated_dsl)
+    translated_yaml_string = convert_to_yaml(translated_dsl)
+    yaml_object = _parse_yaml(translated_yaml_string)
     assert isinstance(yaml_object, dict)
     tabs = yaml_object["tabs"]
     assert isinstance(tabs, list)

From 4f0a0ca79aeb348f98b0acf59212c4f305545383 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 31 Mar 2025 13:27:30 +0200
Subject: [PATCH 095/111] remove unused import

---
 tested/main.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tested/main.py b/tested/main.py
index 096824e1..65aadc37 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -8,7 +8,6 @@
 
 from tested.configs import DodonaConfig, create_bundle
 from tested.dsl import parse_dsl
-from tested.dsl.translate_parser import _convert_dsl
 from tested.nat_translation import run_translation
 from tested.testsuite import parse_test_suite
 

From 84fb41f300890c93028b90ba3db3b89f6270ecb2 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 31 Mar 2025 13:46:06 +0200
Subject: [PATCH 096/111] got rid of the !programming_language tag for this PR

---
 tested/dsl/schema-strict-nat-translation.json | 64 ++++++-------------
 tested/nat_translation.py                     |  2 +-
 tests/test_preprocess_dsl.py                  |  5 +-
 3 files changed, 21 insertions(+), 50 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 4f23a30f..4cbc836f 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -1253,29 +1253,15 @@
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
-          "type" : "object",
-          "required": [
-            "__tag__",
-            "value"
-          ],
-          "properties" : {
-            "__tag__": {
-              "type" : "string",
-              "description" : "The tag used in the yaml",
-              "const":  "!programming_language"
-            },
-            "value":{
-              "type": "object",
-              "description" : "Programming-language-specific statement or expression.",
-              "minProperties" : 1,
-              "propertyNames" : {
-                "$ref" : "#/definitions/programmingLanguage"
-              },
-              "items" : {
-                "type" : "string",
-                "description" : "A language-specific literal, which will be used verbatim."
-              }
-            }
+          "type": "object",
+          "description" : "Programming-language-specific statement or expression.",
+          "minProperties" : 1,
+          "propertyNames" : {
+            "$ref" : "#/definitions/programmingLanguage"
+          },
+          "items" : {
+            "type" : "string",
+            "description" : "A language-specific literal, which will be used verbatim."
           }
         }
       ]
@@ -1287,29 +1273,15 @@
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
-          "type" : "object",
-          "required": [
-            "__tag__",
-            "value"
-          ],
-          "properties" : {
-            "__tag__": {
-              "type" : "string",
-              "description" : "The tag used in the yaml",
-              "const":  "!programming_language"
-            },
-            "value":{
-              "type": "object",
-              "description" : "Programming-language-specific statement or expression.",
-              "minProperties" : 1,
-              "propertyNames" : {
-                "$ref" : "#/definitions/programmingLanguage"
-              },
-              "items" : {
-                "type" : "string",
-                "description" : "A language-specific literal, which will be used verbatim."
-              }
-            }
+          "type": "object",
+          "description" : "Programming-language-specific statement or expression.",
+          "minProperties" : 1,
+          "propertyNames" : {
+            "$ref" : "#/definitions/programmingLanguage"
+          },
+          "items" : {
+            "type" : "string",
+            "description" : "A language-specific literal, which will be used verbatim."
           }
         },
         {
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index d409fd4e..a49047f7 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -41,7 +41,7 @@ def represent_with_tag(self, tag, value):
     @staticmethod
     def custom_representer(dumper, data):
         if "__tag__" in data:
-            if data["__tag__"] != "!programming_language":
+            if data["__tag__"]:
                 return dumper.represent_with_tag(data["__tag__"], data["value"])
             else:
                 data = data["value"]
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index ba5f93bb..532e47f4 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -25,7 +25,6 @@ def validate_natural_translate(yaml_str: str, translated_yaml_str: str):
     yaml_object = parse_yaml(yaml_str)
     translated_dsl = translate_yaml(yaml_object, {}, "en", enviroment)
     translated_yaml = convert_to_yaml(translated_dsl)
-    print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
 
@@ -138,7 +137,7 @@ def test_return():
     validate_natural_translate(yaml_str, translated_yaml_str)
 
 
-def test_nat_lang_and_prog_lang_combo():
+def test_nat_lang_and_prog_lang_combination():
     yaml_str = """
 translations:
   animal:
@@ -151,7 +150,7 @@ def test_nat_lang_and_prog_lang_combo():
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
-      - expression: !programming_language
+      - expression:
           javascript: "{{animal}}_javascript(1 + 1)"
           typescript: "{{animal}}_typescript(1 + 1)"
           java: "Submission.{{animal}}_java(1 + 1)"

From 57fcc0ed6a3dd8cb642a255b19a07b3a342de034 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 31 Mar 2025 14:19:16 +0200
Subject: [PATCH 097/111] Made an extra test

---
 tests/test_preprocess_dsl.py | 37 ++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 532e47f4..5ea7f0e3 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -497,6 +497,43 @@ def test_file_is_generated(mocker: MockerFixture):
     # Check if the file was opened for writing
     mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8")
 
+def test_key_not_found(mocker: MockerFixture):
+    s = mocker.spy(
+        tested.nat_translation, name="generate_new_yaml"  # type: ignore[reportAttributeAccessIssue]
+    )
+
+    mock_files = [
+        mocker.mock_open(read_data=content).return_value
+        for content in [
+            """
+tabs:
+- tab: task3
+  testcases:
+  - statement: !natural_language
+        nl: resultaten = Proberen({{ten}})
+        en: results = Tries({{ten}})
+  - expression: !natural_language
+        nl: tel_woorden(resultaten)
+        en: count_words(results)
+    return: !natural_language
+        nl: Het resultaat is 10
+        en: The result is 10"""
+        ]
+    ]
+    mock_files.append(mocker.mock_open(read_data="{}").return_value)
+    mock_files.append(mocker.mock_open().return_value)
+    mock_opener = mocker.mock_open()
+    mock_opener.side_effect = mock_files
+    mocker.patch("builtins.open", mock_opener)
+
+    _, missing_keys = run_translation(Path("suite.yaml"), "en", True)
+
+    assert missing_keys
+    assert s.call_count == 1
+
+    # Check if the file was opened for writing
+    mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8")
+
 
 def test_parsing_failed():
     yaml_str = """

From c7cde3dc7e3ddf7cb8e1d376d981d63193368e04 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 31 Mar 2025 15:26:28 +0200
Subject: [PATCH 098/111] Fix linting

---
 tests/test_preprocess_dsl.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 5ea7f0e3..10c567a6 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -497,6 +497,7 @@ def test_file_is_generated(mocker: MockerFixture):
     # Check if the file was opened for writing
     mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8")
 
+
 def test_key_not_found(mocker: MockerFixture):
     s = mocker.spy(
         tested.nat_translation, name="generate_new_yaml"  # type: ignore[reportAttributeAccessIssue]

From 79210fa8bf72e7cd87636acc1eaccd1fd7a7283a Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 3 Apr 2025 16:17:05 +0200
Subject: [PATCH 099/111] Give warning for each missing key

---
 tested/configs.py         |  6 ++++--
 tested/judge/core.py      | 17 +++++++++--------
 tested/main.py            |  2 +-
 tested/nat_translation.py |  8 ++++----
 4 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/tested/configs.py b/tested/configs.py
index 765e35e7..45fba590 100644
--- a/tested/configs.py
+++ b/tested/configs.py
@@ -129,7 +129,7 @@ class Bundle:
     language: "Language"
     global_config: GlobalConfig
     out: IO
-    translations_missing_key: bool = False
+    translations_missing_key: list[str] = []
 
     @property
     def config(self) -> DodonaConfig:
@@ -208,7 +208,7 @@ def create_bundle(
     output: IO,
     suite: Suite,
     language: str | None = None,
-    translations_missing_key: bool = False,
+    translations_missing_key: list[str] | None = None,
 ) -> Bundle:
     """
     Create a configuration bundle.
@@ -236,6 +236,8 @@ def create_bundle(
         suite=suite,
     )
     lang_config = langs.get_language(global_config, language)
+    if translations_missing_key is None:
+        translations_missing_key = []
     return Bundle(
         language=lang_config,
         global_config=global_config,
diff --git a/tested/judge/core.py b/tested/judge/core.py
index d5e7b8c4..a2353eb5 100644
--- a/tested/judge/core.py
+++ b/tested/judge/core.py
@@ -119,14 +119,15 @@ def judge(bundle: Bundle):
     collector = OutputManager(bundle.out)
     collector.add(StartJudgement())
     if bundle.translations_missing_key:
-        collector.add_messages(
-            [
-                ExtendedMessage(
-                    "The natural translator found a key that was not defined in the corresponding translations maps!",
-                    permission=Permission.STAFF,
-                )
-            ]
-        )
+        for key in bundle.translations_missing_key:
+            collector.add_messages(
+                [
+                    ExtendedMessage(
+                        f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!",
+                        permission=Permission.STAFF,
+                    )
+                ]
+            )
     max_time = float(bundle.config.time_limit) * 0.9
     start = time.perf_counter()
 
diff --git a/tested/main.py b/tested/main.py
index 65aadc37..f4df3659 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -39,7 +39,7 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
             suite = parse_dsl(textual_suite)
         else:
             suite = parse_test_suite(textual_suite)
-        missing_keys = False
+        missing_keys = []
     else:
         translated_yaml, missing_keys = run_translation(
             Path(f"{config.resources}/{config.test_suite}"),
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index a49047f7..0d1dea84 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -104,11 +104,11 @@ def wrap_in_braces(value):
 
 
 class TrackingUndefined(Undefined):
-    missing_keys = set()
+    missing_keys = list()
 
     def __str__(self):
         # Store the missing key name
-        TrackingUndefined.missing_keys.add(self._undefined_name)
+        TrackingUndefined.missing_keys.append(self._undefined_name)
         # Return it in Jinja syntax to keep it in the template
         return f"{{{{ {self._undefined_name} }}}}"
 
@@ -151,7 +151,7 @@ def parse_yaml(yaml_stream: str) -> Any:
 
 def run_translation(
     path: Path, language: str, to_file: bool = True
-) -> tuple[str, bool]:
+) -> tuple[str, list]:
     try:
         with open(path, "r") as stream:
             yaml_stream = stream.read()
@@ -169,7 +169,7 @@ def run_translation(
     enviroment = create_enviroment()
     translated_data = translate_yaml(parsed_yaml, {}, language, enviroment)
 
-    missing_keys = len(TrackingUndefined.missing_keys) > 0
+    missing_keys = TrackingUndefined.missing_keys
     translated_yaml_string = convert_to_yaml(translated_data)
     if to_file:
         generate_new_yaml(path, translated_yaml_string, language)

From 2c9bc0683e4338ccc8d9de973d3203213e326a07 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 14 Apr 2025 15:18:39 +0200
Subject: [PATCH 100/111] added a bit of documentation and did some last
 changes to Json schema

---
 tested/dsl/schema-strict-nat-translation.json | 50 +++++++++++++++----
 tested/nat_translation.py                     | 25 ++++++++++
 2 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 4cbc836f..167e0e14 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -250,7 +250,7 @@
           ],
           "description" : "The name of this tab."
         },
-        "translation" : {
+        "translations" : {
           "type" : "object",
           "description": "Define translations in the tab scope."
         },
@@ -331,12 +331,14 @@
           "description" : "Defines if the unit/tab is hidden for the student or not"
         },
         "unit" : {
-          "anyOf" : [
+          "oneOf" : [
             {
-              "type" : "string"
+              "type" : "string",
+              "description" : "The name of this tab."
             },
             {
               "type" : "object",
+              "description" : "The name of this tab.",
               "required": [
                 "__tag__",
                 "value"
@@ -394,7 +396,7 @@
       ]
     },
     "_contextList" : {
-      "anyOf" : [
+      "oneOf" : [
         {
           "type" : "array",
           "minItems" : 1,
@@ -429,7 +431,7 @@
       ]
     },
     "_caseList" : {
-      "anyOf" : [
+      "oneOf" : [
         {
           "type" : "array",
           "minItems" : 1,
@@ -464,7 +466,7 @@
       ]
     },
     "_testcaseList" : {
-      "anyOf" : [
+      "oneOf" : [
         {
           "type" : "array",
           "minItems" : 1,
@@ -499,7 +501,7 @@
       ]
     },
     "_scriptList" : {
-      "anyOf" : [
+      "oneOf" : [
         {
           "type" : "array",
           "minItems" : 1,
@@ -1250,6 +1252,7 @@
       "oneOf" : [
         {
           "type" : "string",
+          "format" : "tested-dsl-expression",
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
@@ -1270,6 +1273,7 @@
       "oneOf" : [
         {
           "type" : "string",
+          "format" : "tested-dsl-expression",
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
@@ -1280,8 +1284,33 @@
             "$ref" : "#/definitions/programmingLanguage"
           },
           "items" : {
-            "type" : "string",
-            "description" : "A language-specific literal, which will be used verbatim."
+            "oneOf" : [
+              {
+                "type" : "string",
+                "description" : "A language-specific literal, which will be used verbatim."
+              },
+              {
+                "type" : "object",
+                "required": [
+                  "__tag__",
+                  "value"
+                ],
+                "properties" : {
+                  "__tag__": {
+                    "type" : "string",
+                    "description" : "The tag used in the yaml",
+                    "const":  "!natural_language"
+                  },
+                  "value":{
+                    "type": "object",
+                    "additionalProperties": {
+                      "type" : "string",
+                      "description" : "A language-specific literal, which will be used verbatim."
+                    }
+                  }
+                }
+              }
+            ]
           }
         },
         {
@@ -1903,7 +1932,8 @@
         "kotlin",
         "python",
         "runhaskell",
-        "csharp"
+        "csharp",
+        "cpp"
       ]
     },
     "message" : {
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 0d1dea84..9e13be05 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -40,6 +40,12 @@ def represent_with_tag(self, tag, value):
 
     @staticmethod
     def custom_representer(dumper, data):
+        """
+        Will turn the given object back into YAML.
+
+        :param dumper: The dumper to use.
+        :param data: The object to represent.
+        """
         if "__tag__" in data:
             if data["__tag__"]:
                 return dumper.represent_with_tag(data["__tag__"], data["value"])
@@ -52,7 +58,13 @@ def custom_representer(dumper, data):
 
 
 def construct_custom(loader, tag_suffix, node):
+    """
+    This constructor will turn the given YAML into an object that can be used for translation.
 
+    :param loader: The YAML loader.
+    :param tag_suffix: The tag that was found.
+    :param node: The node to construct.
+    """
     type2method = {
         MappingNode: loader.construct_mapping,
         ScalarNode: loader.construct_scalar,
@@ -75,6 +87,15 @@ def construct_custom(loader, tag_suffix, node):
 def translate_yaml(
     data: Any, translations: dict, language: str, env: Environment
 ) -> Any:
+    """
+    This function will translate the multilingual object.
+
+    :param data: The object to translate.
+    :param translations: The merge of all found translations maps.
+    :param language: The language to translate to.
+    :param env: The Jinja-environment to use.
+    :return: The translated object.
+    """
     if isinstance(data, dict):
         if "__tag__" in data and data["__tag__"] == "!natural_language":
             return translate_yaml(data["value"][language], translations, language, env)
@@ -100,6 +121,10 @@ def translate_yaml(
 
 
 def wrap_in_braces(value):
+    """
+    This function will provide the ability to still keep the curly bracket around the
+    translated result. Example: {{ key | braces }} => {sleutel} and {{ key }} => sleutel.
+    """
     return f"{{{value}}}"
 
 

From 502f51dfeb0c1bee67e5a3b650a5299545fc784c Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 14 Apr 2025 16:20:44 +0200
Subject: [PATCH 101/111] removing redundant line

---
 tested/nat_translation.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 9e13be05..fbd0cf67 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -47,10 +47,8 @@ def custom_representer(dumper, data):
         :param data: The object to represent.
         """
         if "__tag__" in data:
-            if data["__tag__"]:
-                return dumper.represent_with_tag(data["__tag__"], data["value"])
-            else:
-                data = data["value"]
+            return dumper.represent_with_tag(data["__tag__"], data["value"])
+
 
         return dumper.represent_mapping(
             yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data

From 52537f698a3dfaa59cde3ab526a6c1408085069b Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 14 Apr 2025 16:34:04 +0200
Subject: [PATCH 102/111] fixed linting

---
 tested/nat_translation.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index fbd0cf67..543860be 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -49,7 +49,6 @@ def custom_representer(dumper, data):
         if "__tag__" in data:
             return dumper.represent_with_tag(data["__tag__"], data["value"])
 
-
         return dumper.represent_mapping(
             yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data
         )

From 511d6cf3915e8e63a77346f52de342778b4a5ae7 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 14 Apr 2025 17:31:26 +0200
Subject: [PATCH 103/111] cleaned up code some more and added asserts

---
 tested/nat_translation.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 543860be..4915def2 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -95,23 +95,25 @@ def translate_yaml(
     """
     if isinstance(data, dict):
         if "__tag__" in data and data["__tag__"] == "!natural_language":
-            return translate_yaml(data["value"][language], translations, language, env)
+            value = data["value"]
+            assert language in value
+            return translate_yaml(value[language], translations, language, env)
 
         current_translations = data.pop("translations", {})
         for key, value in current_translations.items():
+            assert language in value
             current_translations[key] = value[language]
-        merged_translations = {**translations, **current_translations}
+        translations = {**translations, **current_translations}
 
         return {
-            key: translate_yaml(value, merged_translations, language, env)
+            key: translate_yaml(value, translations, language, env)
             for key, value in data.items()
         }
     elif isinstance(data, list):
         return [translate_yaml(item, translations, language, env) for item in data]
     elif isinstance(data, str):
         try:
-            result = env.from_string(data).render(translations)
-            return result
+            return env.from_string(data).render(translations)
         except TemplateSyntaxError:
             return data
     return data

From c11df0332c884ccb31f9855893c7f98939ee9b9b Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 23 Apr 2025 19:55:02 +0200
Subject: [PATCH 104/111] changing main

---
 tested/main.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tested/main.py b/tested/main.py
index f4df3659..3d575bc6 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -32,21 +32,21 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
                 "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
             )
             raise e
-
-        _, ext = os.path.splitext(config.test_suite)
-        is_yaml = ext.lower() in (".yaml", ".yml")
-        if is_yaml:
-            suite = parse_dsl(textual_suite)
-        else:
-            suite = parse_test_suite(textual_suite)
-        missing_keys = []
     else:
-        translated_yaml, missing_keys = run_translation(
+        textual_suite, missing_keys = run_translation(
             Path(f"{config.resources}/{config.test_suite}"),
             language=language,
             to_file=False,
         )
-        suite = parse_dsl(translated_yaml)
+
+    _, ext = os.path.splitext(config.test_suite)
+    is_yaml = ext.lower() in (".yaml", ".yml")
+    if is_yaml:
+        suite = parse_dsl(textual_suite)
+    else:
+        suite = parse_test_suite(textual_suite)
+    missing_keys = []
+
     pack = create_bundle(
         config, judge_output, suite, translations_missing_key=missing_keys
     )

From f327e19f2982b4aa16dedc5708c6db63908b20b2 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 25 Apr 2025 19:44:04 +0200
Subject: [PATCH 105/111] fixed a few issues

---
 tested/configs.py              | 10 ++++++++--
 tested/dsl/translate_parser.py | 20 ++++++++++++++++++-
 tested/judge/core.py           | 12 ++----------
 tested/main.py                 | 35 ++++++++++++++++------------------
 tested/nat_translation.py      | 32 ++++++++++++++++++-------------
 5 files changed, 64 insertions(+), 45 deletions(-)

diff --git a/tested/configs.py b/tested/configs.py
index 45fba590..e15d133d 100644
--- a/tested/configs.py
+++ b/tested/configs.py
@@ -8,6 +8,8 @@
 
 from attrs import define, evolve, field
 
+from tested.dodona import ExtendedMessage
+from tested.dsl.translate_parser import build_preprocessor_messages
 from tested.parsing import fallback_field, get_converter
 from tested.testsuite import ExecutionMode, Suite, SupportedLanguage
 from tested.utils import get_identifier, smart_close
@@ -129,7 +131,7 @@ class Bundle:
     language: "Language"
     global_config: GlobalConfig
     out: IO
-    translations_missing_key: list[str] = []
+    preprocessor_messages: list[ExtendedMessage] = []
 
     @property
     def config(self) -> DodonaConfig:
@@ -236,11 +238,15 @@ def create_bundle(
         suite=suite,
     )
     lang_config = langs.get_language(global_config, language)
+
+    translations_missing_key = []
     if translations_missing_key is None:
         translations_missing_key = []
+
+    messages = build_preprocessor_messages(translations_missing_key)
     return Bundle(
         language=lang_config,
         global_config=global_config,
         out=output,
-        translations_missing_key=translations_missing_key,
+        preprocessor_messages=messages,
     )
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 1e26a991..b50e7891 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -28,7 +28,7 @@
     StringTypes,
     resolve_to_basic,
 )
-from tested.dodona import ExtendedMessage
+from tested.dodona import ExtendedMessage, Permission
 from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string
 from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error
 from tested.parsing import get_converter, suite_to_json
@@ -92,6 +92,24 @@ class ReturnOracle(dict):
 )
 
 
+def build_preprocessor_messages(
+    translations_missing_key: list[str],
+) -> list[ExtendedMessage]:
+    """
+    Build the preprocessor messages from the missing keys.
+
+    :param translations_missing_key: The missing keys.
+    :return: The preprocessor messages.
+    """
+    return [
+        ExtendedMessage(
+            f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!",
+            permission=Permission.STAFF,
+        )
+        for key in translations_missing_key
+    ]
+
+
 def _convert_language_dictionary(
     original: dict[str, str]
 ) -> dict[SupportedLanguage, str]:
diff --git a/tested/judge/core.py b/tested/judge/core.py
index a2353eb5..e07013a3 100644
--- a/tested/judge/core.py
+++ b/tested/judge/core.py
@@ -118,16 +118,8 @@ def judge(bundle: Bundle):
     # Do the set-up for the judgement.
     collector = OutputManager(bundle.out)
     collector.add(StartJudgement())
-    if bundle.translations_missing_key:
-        for key in bundle.translations_missing_key:
-            collector.add_messages(
-                [
-                    ExtendedMessage(
-                        f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!",
-                        permission=Permission.STAFF,
-                    )
-                ]
-            )
+    if bundle.preprocessor_messages:
+        collector.add_messages(bundle.preprocessor_messages)
     max_time = float(bundle.config.time_limit) * 0.9
     start = time.perf_counter()
 
diff --git a/tested/main.py b/tested/main.py
index 3d575bc6..8178b17c 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -8,7 +8,7 @@
 
 from tested.configs import DodonaConfig, create_bundle
 from tested.dsl import parse_dsl
-from tested.nat_translation import run_translation
+from tested.nat_translation import run_translation_with_str
 from tested.testsuite import parse_test_suite
 
 
@@ -20,32 +20,29 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
     :param judge_output: Where the judge output will be written to.
     :param language: The language to use to translate the test-suite.
     """
-    if language is None:
-        try:
-            with open(f"{config.resources}/{config.test_suite}", "r") as t:
-                textual_suite = t.read()
-        except FileNotFoundError as e:
-            print(
-                "The test suite was not found. Check your exercise's config.json file."
-            )
-            print(
-                "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
-            )
-            raise e
-    else:
-        textual_suite, missing_keys = run_translation(
-            Path(f"{config.resources}/{config.test_suite}"),
-            language=language,
-            to_file=False,
+    missing_keys = []
+    try:
+        with open(f"{config.resources}/{config.test_suite}", "r") as t:
+            textual_suite = t.read()
+    except FileNotFoundError as e:
+        print("The test suite was not found. Check your exercise's config.json file.")
+        print(
+            "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
         )
+        raise e
 
     _, ext = os.path.splitext(config.test_suite)
     is_yaml = ext.lower() in (".yaml", ".yml")
     if is_yaml:
+        if language:
+            textual_suite, missing_keys = run_translation_with_str(
+                textual_suite,
+                language=language,
+                to_file=False,
+            )
         suite = parse_dsl(textual_suite)
     else:
         suite = parse_test_suite(textual_suite)
-    missing_keys = []
 
     pack = create_bundle(
         config, judge_output, suite, translations_missing_key=missing_keys
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 4915def2..07f646b2 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -173,20 +173,9 @@ def parse_yaml(yaml_stream: str) -> Any:
         raise_yaml_error(yaml_stream, exc)
 
 
-def run_translation(
-    path: Path, language: str, to_file: bool = True
+def run_translation_with_str(
+    yaml_stream: str, language: str, to_file: bool = True, path: Path = None
 ) -> tuple[str, list]:
-    try:
-        with open(path, "r") as stream:
-            yaml_stream = stream.read()
-    except FileNotFoundError as e:
-        print("The test suite was not found. Check your exercise's config.json file.")
-        print(
-            "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
-        )
-        raise e
-    _, ext = os.path.splitext(path)
-    assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}."
     parsed_yaml = parse_yaml(yaml_stream)
     validate_pre_dsl(parsed_yaml)
 
@@ -202,6 +191,23 @@ def run_translation(
         return translated_yaml_string, missing_keys
 
 
+def run_translation(
+    path: Path, language: str, to_file: bool = True
+) -> tuple[str, list]:
+    try:
+        with open(path, "r") as stream:
+            yaml_stream = stream.read()
+    except FileNotFoundError as e:
+        print("The test suite was not found. Check your exercise's config.json file.")
+        print(
+            "Remember that the test suite is a path relative to the 'evaluation' folder of your exercise."
+        )
+        raise e
+    _, ext = os.path.splitext(path)
+    assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}."
+    return run_translation_with_str(yaml_stream, language, to_file, path)
+
+
 if __name__ == "__main__":
     n = len(sys.argv)
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."

From 5cd9e15a69cda7bded54631d7c86f67acc5c340e Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 25 Apr 2025 19:46:55 +0200
Subject: [PATCH 106/111] remove unused import

---
 tested/main.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tested/main.py b/tested/main.py
index 8178b17c..4abdd3c4 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -3,7 +3,6 @@
 """
 
 import os
-from pathlib import Path
 from typing import IO
 
 from tested.configs import DodonaConfig, create_bundle

From 1d407767db0baf10c2f8219d167ed032fc05ffa0 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 25 Apr 2025 19:54:57 +0200
Subject: [PATCH 107/111] fixing typing issue

---
 tested/nat_translation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 07f646b2..26fc1307 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -174,7 +174,7 @@ def parse_yaml(yaml_stream: str) -> Any:
 
 
 def run_translation_with_str(
-    yaml_stream: str, language: str, to_file: bool = True, path: Path = None
+    yaml_stream: str, language: str, to_file: bool = True, path: Path | None = None
 ) -> tuple[str, list]:
     parsed_yaml = parse_yaml(yaml_stream)
     validate_pre_dsl(parsed_yaml)
@@ -184,7 +184,7 @@ def run_translation_with_str(
 
     missing_keys = TrackingUndefined.missing_keys
     translated_yaml_string = convert_to_yaml(translated_data)
-    if to_file:
+    if to_file and path is not None:
         generate_new_yaml(path, translated_yaml_string, language)
         return "", missing_keys
     else:

From 3a97b15f4d81a2c4326fe5fbdf662ec7931a3a46 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 28 Apr 2025 12:53:17 +0200
Subject: [PATCH 108/111] made few changes

---
 tested/configs.py              | 14 ++++-----
 tested/dsl/dsl_errors.py       | 20 +++++++++++++
 tested/dsl/translate_parser.py | 20 +------------
 tested/judge/core.py           |  2 --
 tested/main.py                 | 14 +++------
 tested/nat_translation.py      | 30 ++++++++++---------
 tests/test_preprocess_dsl.py   | 54 +++++++++-------------------------
 7 files changed, 60 insertions(+), 94 deletions(-)

diff --git a/tested/configs.py b/tested/configs.py
index e15d133d..ea751539 100644
--- a/tested/configs.py
+++ b/tested/configs.py
@@ -9,7 +9,6 @@
 from attrs import define, evolve, field
 
 from tested.dodona import ExtendedMessage
-from tested.dsl.translate_parser import build_preprocessor_messages
 from tested.parsing import fallback_field, get_converter
 from tested.testsuite import ExecutionMode, Suite, SupportedLanguage
 from tested.utils import get_identifier, smart_close
@@ -210,7 +209,7 @@ def create_bundle(
     output: IO,
     suite: Suite,
     language: str | None = None,
-    translations_missing_key: list[str] | None = None,
+    preprocessor_messages: list[ExtendedMessage] | None = None,
 ) -> Bundle:
     """
     Create a configuration bundle.
@@ -220,7 +219,7 @@ def create_bundle(
     :param suite: The test suite.
     :param language: Optional programming language. If None, the one from the Dodona
                      configuration will be used.
-    :param translations_missing_key: Indicator that the natural language translator
+    :param preprocessor_messages: Indicator that the natural language translator
     for the DSL key that was not defined in any translations map.
 
     :return: The configuration bundle.
@@ -238,15 +237,12 @@ def create_bundle(
         suite=suite,
     )
     lang_config = langs.get_language(global_config, language)
+    if preprocessor_messages is None:
+        preprocessor_messages = []
 
-    translations_missing_key = []
-    if translations_missing_key is None:
-        translations_missing_key = []
-
-    messages = build_preprocessor_messages(translations_missing_key)
     return Bundle(
         language=lang_config,
         global_config=global_config,
         out=output,
-        preprocessor_messages=messages,
+        preprocessor_messages=preprocessor_messages,
     )
diff --git a/tested/dsl/dsl_errors.py b/tested/dsl/dsl_errors.py
index 6f1c4d93..f5949729 100644
--- a/tested/dsl/dsl_errors.py
+++ b/tested/dsl/dsl_errors.py
@@ -4,6 +4,8 @@
 import yaml
 from jsonschema.exceptions import ValidationError
 
+from tested.dodona import ExtendedMessage, Permission
+
 
 class InvalidYamlError(ValueError):
     pass
@@ -82,3 +84,21 @@ def raise_yaml_error(yaml_stream: str, exc: yaml.MarkedYAMLError):
         )
     )
     raise exc
+
+
+def build_preprocessor_messages(
+    translations_missing_key: list[str],
+) -> list[ExtendedMessage]:
+    """
+    Build the preprocessor messages from the missing keys.
+
+    :param translations_missing_key: The missing keys.
+    :return: The preprocessor messages.
+    """
+    return [
+        ExtendedMessage(
+            f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!",
+            permission=Permission.STAFF,
+        )
+        for key in translations_missing_key
+    ]
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index b50e7891..1e26a991 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -28,7 +28,7 @@
     StringTypes,
     resolve_to_basic,
 )
-from tested.dodona import ExtendedMessage, Permission
+from tested.dodona import ExtendedMessage
 from tested.dsl.ast_translator import InvalidDslError, extract_comment, parse_string
 from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error
 from tested.parsing import get_converter, suite_to_json
@@ -92,24 +92,6 @@ class ReturnOracle(dict):
 )
 
 
-def build_preprocessor_messages(
-    translations_missing_key: list[str],
-) -> list[ExtendedMessage]:
-    """
-    Build the preprocessor messages from the missing keys.
-
-    :param translations_missing_key: The missing keys.
-    :return: The preprocessor messages.
-    """
-    return [
-        ExtendedMessage(
-            f"The natural translator found the key {key}, that was not defined in the corresponding translations maps!",
-            permission=Permission.STAFF,
-        )
-        for key in translations_missing_key
-    ]
-
-
 def _convert_language_dictionary(
     original: dict[str, str]
 ) -> dict[SupportedLanguage, str]:
diff --git a/tested/judge/core.py b/tested/judge/core.py
index e07013a3..811b21d7 100644
--- a/tested/judge/core.py
+++ b/tested/judge/core.py
@@ -10,9 +10,7 @@
     CloseContext,
     CloseJudgement,
     CloseTab,
-    ExtendedMessage,
     Metadata,
-    Permission,
     StartContext,
     StartJudgement,
     StartTab,
diff --git a/tested/main.py b/tested/main.py
index 4abdd3c4..e72df435 100644
--- a/tested/main.py
+++ b/tested/main.py
@@ -7,7 +7,7 @@
 
 from tested.configs import DodonaConfig, create_bundle
 from tested.dsl import parse_dsl
-from tested.nat_translation import run_translation_with_str
+from tested.nat_translation import apply_translations
 from tested.testsuite import parse_test_suite
 
 
@@ -19,7 +19,7 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
     :param judge_output: Where the judge output will be written to.
     :param language: The language to use to translate the test-suite.
     """
-    missing_keys = []
+    messages = []
     try:
         with open(f"{config.resources}/{config.test_suite}", "r") as t:
             textual_suite = t.read()
@@ -34,18 +34,12 @@ def run(config: DodonaConfig, judge_output: IO, language: str | None = None):
     is_yaml = ext.lower() in (".yaml", ".yml")
     if is_yaml:
         if language:
-            textual_suite, missing_keys = run_translation_with_str(
-                textual_suite,
-                language=language,
-                to_file=False,
-            )
+            textual_suite, messages = apply_translations(textual_suite, language)
         suite = parse_dsl(textual_suite)
     else:
         suite = parse_test_suite(textual_suite)
 
-    pack = create_bundle(
-        config, judge_output, suite, translations_missing_key=missing_keys
-    )
+    pack = create_bundle(config, judge_output, suite, preprocessor_messages=messages)
     from .judge import judge
 
     judge(pack)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 26fc1307..560e9f50 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -10,7 +10,12 @@
 from jsonschema.validators import validator_for
 from yaml.nodes import MappingNode, ScalarNode, SequenceNode
 
-from tested.dsl.dsl_errors import handle_dsl_validation_errors, raise_yaml_error
+from tested.dodona import ExtendedMessage
+from tested.dsl.dsl_errors import (
+    build_preprocessor_messages,
+    handle_dsl_validation_errors,
+    raise_yaml_error,
+)
 
 
 def validate_pre_dsl(yaml_object: Any):
@@ -173,9 +178,9 @@ def parse_yaml(yaml_stream: str) -> Any:
         raise_yaml_error(yaml_stream, exc)
 
 
-def run_translation_with_str(
-    yaml_stream: str, language: str, to_file: bool = True, path: Path | None = None
-) -> tuple[str, list]:
+def apply_translations(
+    yaml_stream: str, language: str
+) -> tuple[str, list[ExtendedMessage]]:
     parsed_yaml = parse_yaml(yaml_stream)
     validate_pre_dsl(parsed_yaml)
 
@@ -183,17 +188,12 @@ def run_translation_with_str(
     translated_data = translate_yaml(parsed_yaml, {}, language, enviroment)
 
     missing_keys = TrackingUndefined.missing_keys
+    messages = build_preprocessor_messages(missing_keys)
     translated_yaml_string = convert_to_yaml(translated_data)
-    if to_file and path is not None:
-        generate_new_yaml(path, translated_yaml_string, language)
-        return "", missing_keys
-    else:
-        return translated_yaml_string, missing_keys
+    return translated_yaml_string, messages
 
 
-def run_translation(
-    path: Path, language: str, to_file: bool = True
-) -> tuple[str, list]:
+def translate_file(path: Path, language: str) -> tuple[str, list[ExtendedMessage]]:
     try:
         with open(path, "r") as stream:
             yaml_stream = stream.read()
@@ -205,11 +205,13 @@ def run_translation(
         raise e
     _, ext = os.path.splitext(path)
     assert ext.lower() in (".yaml", ".yml"), f"expected a yaml file, got {ext}."
-    return run_translation_with_str(yaml_stream, language, to_file, path)
+    translated_yaml_string, messages = apply_translations(yaml_stream, language)
+    generate_new_yaml(path, translated_yaml_string, language)
+    return translated_yaml_string, messages
 
 
 if __name__ == "__main__":
     n = len(sys.argv)
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
 
-    run_translation(Path(sys.argv[1]), sys.argv[2])
+    translate_file(Path(sys.argv[1]), sys.argv[2])
diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py
index 10c567a6..34bb5227 100644
--- a/tests/test_preprocess_dsl.py
+++ b/tests/test_preprocess_dsl.py
@@ -4,6 +4,7 @@
 from pytest_mock import MockerFixture
 
 import tested
+from tested.dodona import ExtendedMessage, Permission
 from tested.dsl.translate_parser import (
     ExpressionString,
     ReturnOracle,
@@ -14,7 +15,7 @@
     convert_to_yaml,
     create_enviroment,
     parse_yaml,
-    run_translation,
+    translate_file,
     translate_yaml,
     validate_pre_dsl,
 )
@@ -443,14 +444,15 @@ def test_run_is_correct(mocker: MockerFixture):
         ]
     ]
     mock_files.append(mocker.mock_open(read_data="{}").return_value)
+    mock_files.append(mocker.mock_open().return_value)
     mock_opener = mocker.mock_open()
     mock_opener.side_effect = mock_files
     mocker.patch("builtins.open", mock_opener)
 
-    translated_yaml, _ = run_translation(Path("suite.yaml"), "en", False)
+    translated_yaml, _ = translate_file(Path("suite.yaml"), "en")
     yaml_object = parse_yaml(translated_yaml)
 
-    assert s.call_count == 0
+    assert s.call_count == 1
     assert isinstance(yaml_object, dict)
     tabs = yaml_object["tabs"]
     assert isinstance(tabs, list)
@@ -460,40 +462,6 @@ def test_run_is_correct(mocker: MockerFixture):
         "return": "The result is 10",
     }
 
-
-def test_file_is_generated(mocker: MockerFixture):
-    s = mocker.spy(
-        tested.nat_translation, name="generate_new_yaml"  # type: ignore[reportAttributeAccessIssue]
-    )
-
-    mock_files = [
-        mocker.mock_open(read_data=content).return_value
-        for content in [
-            """
-tabs:
-- tab: task3
-  testcases:
-  - statement: !natural_language
-        nl: resultaten = Proberen(10)
-        en: results = Tries(10)
-  - expression: !natural_language
-        nl: tel_woorden(resultaten)
-        en: count_words(results)
-    return: !natural_language
-        nl: Het resultaat is 10
-        en: The result is 10"""
-        ]
-    ]
-    mock_files.append(mocker.mock_open(read_data="{}").return_value)
-    mock_files.append(mocker.mock_open().return_value)
-    mock_opener = mocker.mock_open()
-    mock_opener.side_effect = mock_files
-    mocker.patch("builtins.open", mock_opener)
-
-    run_translation(Path("suite.yaml"), "en", True)
-
-    assert s.call_count == 1
-
     # Check if the file was opened for writing
     mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8")
 
@@ -527,10 +495,16 @@ def test_key_not_found(mocker: MockerFixture):
     mock_opener.side_effect = mock_files
     mocker.patch("builtins.open", mock_opener)
 
-    _, missing_keys = run_translation(Path("suite.yaml"), "en", True)
+    _, messages = translate_file(Path("suite.yaml"), "en")
 
-    assert missing_keys
+    assert messages
     assert s.call_count == 1
+    assert isinstance(messages[0], ExtendedMessage)
+    assert (
+        messages[0].description
+        == "The natural translator found the key ten, that was not defined in the corresponding translations maps!"
+    )
+    assert messages[0].permission == Permission.STAFF
 
     # Check if the file was opened for writing
     mock_opener.assert_any_call(Path("suite-en.yaml"), "w", encoding="utf-8")
@@ -639,7 +613,7 @@ def test_dumper():
 
 def test_run_is_correct_when_no_file():
     try:
-        run_translation(Path("suite.yaml"), "en", False)
+        translate_file(Path("suite.yaml"), "en")
     except FileNotFoundError:
         print("As expected")
     else:

From bb3438e99263a5ce3137e34669bb1deead22dad2 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Mon, 28 Apr 2025 17:27:07 +0200
Subject: [PATCH 109/111] removed default from arguments

---
 tested/__main__.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tested/__main__.py b/tested/__main__.py
index 72f9f085..ba8779bb 100644
--- a/tested/__main__.py
+++ b/tested/__main__.py
@@ -36,7 +36,6 @@
     "--translate",
     type=str,
     help="Specifies the language to translate translate the dsl to.",
-    default="-",
 )
 parser = parser.parse_args()
 

From f09f925eab542c5ffbbfcc24bfbf0feff2de082a Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 2 May 2025 12:05:06 +0200
Subject: [PATCH 110/111] changed some of the names

---
 tested/nat_translation.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 560e9f50..ecff42c6 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -33,7 +33,7 @@ def validate_pre_dsl(yaml_object: Any):
     handle_dsl_validation_errors(errors)
 
 
-class CustomDumper(yaml.SafeDumper):
+class CustomTagFormatDumper(yaml.SafeDumper):
 
     def represent_with_tag(self, tag, value):
         if isinstance(value, dict):
@@ -44,7 +44,7 @@ def represent_with_tag(self, tag, value):
             return self.represent_scalar(tag, value)
 
     @staticmethod
-    def custom_representer(dumper, data):
+    def custom_tag_format_representer(dumper, data):
         """
         Will turn the given object back into YAML.
 
@@ -59,7 +59,7 @@ def custom_representer(dumper, data):
         )
 
 
-def construct_custom(loader, tag_suffix, node):
+def construct_custom_tag_format(loader, tag_suffix, node):
     """
     This constructor will turn the given YAML into an object that can be used for translation.
 
@@ -159,9 +159,9 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
 
 
 def convert_to_yaml(translated_data: Any) -> str:
-    CustomDumper.add_representer(dict, CustomDumper.custom_representer)
+    CustomTagFormatDumper.add_representer(dict, CustomTagFormatDumper.custom_tag_format_representer)
     return yaml.dump(
-        translated_data, Dumper=CustomDumper, allow_unicode=True, sort_keys=False
+        translated_data, Dumper=CustomTagFormatDumper, allow_unicode=True, sort_keys=False
     )
 
 
@@ -170,7 +170,7 @@ def parse_yaml(yaml_stream: str) -> Any:
     Parse a string or stream to YAML.
     """
     loader: type[yaml.Loader] = cast(type[yaml.Loader], yaml.SafeLoader)
-    yaml.add_multi_constructor("", construct_custom, loader)
+    yaml.add_multi_constructor("", construct_custom_tag_format, loader)
 
     try:
         return yaml.load(yaml_stream, loader)

From 46ea4cced4054a5760fa5edfcb6daba04b8e7ab3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 2 May 2025 12:16:27 +0200
Subject: [PATCH 111/111] fixed linting

---
 tested/nat_translation.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index ecff42c6..65cfcfa5 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -159,9 +159,14 @@ def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
 
 
 def convert_to_yaml(translated_data: Any) -> str:
-    CustomTagFormatDumper.add_representer(dict, CustomTagFormatDumper.custom_tag_format_representer)
+    CustomTagFormatDumper.add_representer(
+        dict, CustomTagFormatDumper.custom_tag_format_representer
+    )
     return yaml.dump(
-        translated_data, Dumper=CustomTagFormatDumper, allow_unicode=True, sort_keys=False
+        translated_data,
+        Dumper=CustomTagFormatDumper,
+        allow_unicode=True,
+        sort_keys=False,
     )