@@ -414,7 +414,7 @@ def _walk_fsm(
414
414
fsm_transitions : Dict [Tuple [int , int ], int ],
415
415
fsm_initial : int ,
416
416
fsm_finals : Set [int ],
417
- token_trans_key_seq : Sequence [int ],
417
+ token_transition_keys : Sequence [int ],
418
418
start_state : int ,
419
419
full_match : bool = True ,
420
420
) -> List [int ]:
@@ -424,7 +424,7 @@ def _walk_fsm(
424
424
425
425
# Iterate over token transition key sequence. The transition key
426
426
# sequence represents the FSM traversal rules of the tokens symbols.
427
- for i , trans_key in enumerate (token_trans_key_seq ):
427
+ for i , trans_key in enumerate (token_transition_keys ):
428
428
new_state = fsm_transitions .get ((state , trans_key ))
429
429
430
430
if new_state is None :
@@ -448,7 +448,7 @@ def _walk_fsm(
448
448
449
449
def walk_fsm (
450
450
fsm : BetterFSM ,
451
- token_trans_key_seq : Sequence [int ],
451
+ token_transition_keys : Sequence [int ],
452
452
start_state : int ,
453
453
full_match : bool = True ,
454
454
) -> List [int ]:
@@ -462,7 +462,7 @@ def walk_fsm(
462
462
463
463
# Iterate over token transition key sequence. The transition key
464
464
# sequence represents the FSM traversal rules of the tokens symbols.
465
- for i , trans_key in enumerate (token_trans_key_seq ):
465
+ for i , trans_key in enumerate (token_transition_keys ):
466
466
new_state = fsm_transitions .get ((state , trans_key ))
467
467
468
468
if new_state is None :
@@ -703,10 +703,10 @@ def get_token_transition_keys(
703
703
alphabet_symbol_mapping .get (symbol , alphabet_anything_value )
704
704
)
705
705
706
- tok_trans_array = np .empty (len (token_transition_keys ), dtype = np .int64 )
706
+ token_transition_keys_array = np .empty (len (token_transition_keys ), dtype = np .int64 )
707
707
for j in range (len (token_transition_keys )):
708
- tok_trans_array [j ] = token_transition_keys [j ]
709
- return tok_trans_array
708
+ token_transition_keys_array [j ] = token_transition_keys [j ]
709
+ return token_transition_keys_array
710
710
711
711
712
712
@numba .njit (cache = True , nogil = True )
@@ -718,14 +718,14 @@ def get_vocabulary_transition_keys(
718
718
"""
719
719
Calculate the sequence transition keys for each token str within a vocabulary
720
720
"""
721
- tokens_trans_keys = numba .typed .List .empty_list (numba .int64 [:])
721
+ vocab_transition_keys = numba .typed .List .empty_list (numba .int64 [:])
722
722
for token_str , _ in vocabulary :
723
- trans_key_seq_array = get_token_transition_keys (
723
+ token_transition_keys = get_token_transition_keys (
724
724
alphabet_symbol_mapping , alphabet_anything_value , token_str
725
725
)
726
- tokens_trans_keys .append (trans_key_seq_array )
726
+ vocab_transition_keys .append (token_transition_keys )
727
727
728
- return tokens_trans_keys
728
+ return vocab_transition_keys
729
729
730
730
731
731
def create_fsm_index_end_to_end (
0 commit comments