From 69b17d3ceea1df842866769159234b748724b68a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aalok=20=7C=20=E0=A4=86=E0=A4=B2=E0=A5=8B=E0=A4=95?= <10784697+aalok-sathe@users.noreply.github.com> Date: Thu, 21 Apr 2022 13:56:03 -0400 Subject: [PATCH] Add default tokenizer for gpt_neox (the same as gpt_neo) --- src/transformers/models/auto/tokenization_auto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index a60484134096..fc015fc69692 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -172,6 +172,7 @@ ("wav2vec2", ("Wav2Vec2CTCTokenizer", None)), ("hubert", ("Wav2Vec2CTCTokenizer", None)), ("gpt_neo", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), + ("gpt_neox", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), ("luke", ("LukeTokenizer", None)), ("mluke", ("MLukeTokenizer" if is_sentencepiece_available() else None, None)), ("bigbird_pegasus", ("PegasusTokenizer", "PegasusTokenizerFast" if is_tokenizers_available() else None)),