Merge pull request #584 from sillsdev/issue365

Solve Issue #365 and #390

Author: AmeWenJ

silnlp/nmt/config.py

@@ -493,7 +493,8 @@ def preprocess(self, stats: bool, force_align: bool = False) -> None:
                 LOGGER.error(f"The source file {str(file)} does not exist.")
                 return
 
-        self._build_vocabs(stats)
+        if self.data["tokenize"]:
+            self._build_vocabs(stats)
         tokenizer = self.create_tokenizer()
         self._build_corpora(tokenizer, stats, force_align)
         LOGGER.info("Preprocessing completed")
@@ -557,7 +558,7 @@ def _build_corpora(self, tokenizer: Tokenizer, stats: bool, force_align: bool) -
             dict_count = self._write_dictionary(tokenizer, src_terms_files, trg_terms_files)
             LOGGER.info(f"dictionary size: {dict_count}")
 
-        if stats:
+        if stats and self.data["tokenize"]:
             self._calculate_tokenization_stats()
 
         return train_count

silnlp/nmt/hugging_face_config.py

@@ -540,7 +540,7 @@ def _build_vocabs(self, stats: bool = False) -> None:
                 ["Target", 0],
             ]
 
-        if stats:
+        if stats and self.data["tokenize"]:
             stats_columns = pd.MultiIndex.from_tuples(
                 [
                     (" ", "Translation Side"),