feat: Add test for tokenizer

soeque1 · Feb 1, 2021 · 8f29633 · 8f29633
1 parent 41f3e30
commit 8f29633
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 1 deletion.
diff --git a/cfgs/pipelines/word_piece_with_morpheme.yaml b/cfgs/pipelines/word_piece_with_morpheme.yaml
@@ -9,7 +9,7 @@ Path:
 
 
 Pipelines:
-    Tokenizer: WordPieceTokenizer()
+    Tokenizer: WordPieceTokenizer(unk_token='[UNK]')
 
     normalizer: []
 

diff --git a/prepare.sh b/prepare.sh
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -0,0 +1,11 @@
+from tokenizers import normalizers
+
+
+def test_tokenizer(cfg):
+    tokenizer = cfg['Pipelines']['Tokenizer']
+    tokenizer.pre_tokenizer = cfg['Pipelines']['pre_tokenizer']
+    tokenizer.normalizer = normalizers.Sequence(cfg['Pipelines']['normalizer'])
+    tokenizer.decoder = cfg['Pipelines']['decoder']
+
+    tokenizer.train_from_iterator(['안녕하세요'])
+    assert tokenizer.encode('안녕').tokens == ['안', '##녕']