update run_evaluate script for cased itn (#164)

* update run_evaluate script for cased itn

Signed-off-by: Mariana Graterol Fuenmayor <[email protected]>

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Signed-off-by: Mariana Graterol Fuenmayor <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: mgrafu

nemo_text_processing/inverse_text_normalization/run_evaluate.py

@@ -33,8 +33,14 @@ def parse_args():
     parser = ArgumentParser()
     parser.add_argument("--input", help="input file path", type=str)
     parser.add_argument(
-        "--lang", help="language", choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'vi', 'hy'], default="en", type=str
+        "--lang",
+        help="language",
+        choices=["ar", "de", "en", "es", "es_en", "fr", "hy", "mr", "pt", "ru", "sv", "vi", "zh"],
+        default="en",
+        type=str,
     )
+    parser.add_argument("--input_case", choices=["lower_cased", "cased"])
+    parser.add_argument("--output_case", choices=["lower_cased", "cased"])
     parser.add_argument(
         "--cat",
         dest="category",
@@ -54,10 +60,15 @@ def parse_args():
     if args.lang == 'en':
         from nemo_text_processing.inverse_text_normalization.en.clean_eval_data import filter_loaded_data
     file_path = args.input
-    inverse_normalizer = InverseNormalizer(lang=args.lang)
+    inverse_normalizer = InverseNormalizer(lang=args.lang, input_case=args.input_case)
 
     print("Loading training data: " + file_path)
-    training_data = load_files([file_path])
+    if args.output_case == "lower_cased":
+        to_lower = True
+    elif args.output_case == "cased":
+        to_lower = False
+
+    training_data = load_files([file_path], to_lower=to_lower)
 
     if args.filter:
         training_data = filter_loaded_data(training_data)

nemo_text_processing/text_normalization/data_loader_utils.py

@@ -46,7 +46,7 @@
 ]
 
 
-def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
+def _load_kaggle_text_norm_file(file_path: str, to_lower: bool) -> List[Instance]:
     """
     https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish
     Loads text file in the Kaggle Google text normalization file format: <semiotic class>\t<unnormalized text>\t<`self` if trivial class or normalized text>
@@ -76,8 +76,9 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
                 res.append(Instance(token_type=EOS_TYPE, un_normalized="", normalized=""))
             else:
                 l_type, l_token, l_normalized = parts
-                l_token = l_token.lower()
-                l_normalized = l_normalized.lower()
+                if to_lower:
+                    l_token = l_token.lower()
+                    l_normalized = l_normalized.lower()
 
                 if l_type == PLAIN_TYPE:
                     res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_token))
@@ -86,7 +87,7 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
     return res
 
 
-def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) -> List[Instance]:
+def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file, to_lower: bool = True) -> List[Instance]:
     """
     Load given list of text files using the `load_func` function.
 
@@ -98,7 +99,7 @@ def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) ->
     """
     res = []
     for file_path in file_paths:
-        res.extend(load_func(file_path=file_path))
+        res.extend(load_func(file_path=file_path, to_lower=to_lower))
     return res