Revert the previous two commits because I forgot to create a new branch :(

Author: AmeWenJ

silnlp/nmt/config.py

@@ -493,9 +493,7 @@ def preprocess(self, stats: bool, force_align: bool = False) -> None:
                 LOGGER.error(f"The source file {str(file)} does not exist.")
                 return
 
-        if self.data["tokenize"]:
-            self._build_vocabs(stats)
-
+        self._build_vocabs(stats)
         tokenizer = self.create_tokenizer()
         self._build_corpora(tokenizer, stats, force_align)
         LOGGER.info("Preprocessing completed")
@@ -559,7 +557,7 @@ def _build_corpora(self, tokenizer: Tokenizer, stats: bool, force_align: bool) -
             dict_count = self._write_dictionary(tokenizer, src_terms_files, trg_terms_files)
             LOGGER.info(f"dictionary size: {dict_count}")
 
-        if stats and self.data["tokenize"]:
+        if stats:
             self._calculate_tokenization_stats()
 
         return train_count

silnlp/nmt/hugging_face_config.py

@@ -540,7 +540,7 @@ def _build_vocabs(self, stats: bool = False) -> None:
                 ["Target", 0],
             ]
 
-        if stats and self.data["tokenize"]:
+        if stats:
             stats_columns = pd.MultiIndex.from_tuples(
                 [
                     (" ", "Translation Side"),