Skip to content

Commit ae5d537

Browse files
committed
add comment on get_component
1 parent ed0e8aa commit ae5d537

File tree

1 file changed

+4
-0
lines changed

1 file changed

+4
-0
lines changed

tests/models/chameleon/test_processor_chameleon.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ class ChameleonProcessorTest(ProcessorTesterMixin, unittest.TestCase):
1313

1414
def get_component(self, attribute, **kwargs):
1515
assert attribute in self.processor_class.attributes
16+
if attribute != "tokenizer":
17+
return super().get_component(attribute, **kwargs)
18+
# We use the fast tokenizer by default as the slow tokenizer expects the vocab file to be present in the loading directory.
19+
# This vocab file is neither in the official repo nor does it get saved when we save the processor in `setUp` below.
1620
component_class_name = getattr(self.processor_class, f"{attribute}_class")
1721
if isinstance(component_class_name, tuple):
1822
if "_fast" in component_class_name[0]:

0 commit comments

Comments
 (0)