Use more performant fsm backend

Author: lapp0

benchmarks/bench_cfg_guide.py

@@ -7,8 +7,6 @@
 from outlines.fsm.guide import CFGGuide
 from outlines.models.transformers import TransformerTokenizer
 
-from .common import ensure_numba_compiled
-
 random.seed(42)
 
 
@@ -30,9 +28,6 @@ class CFGGuideBenchmark:
 
     def setup(self, grammar_name):
         self.tokenizer = get_tiny_tokenizer()
-        ensure_numba_compiled(
-            self.tokenizer
-        )  # numba not currently used, but will be in the future
         self.prebuilt_cfg_guide = CFGGuide(
             benched_grammars[grammar_name], self.tokenizer
         )

benchmarks/bench_json_schema.py

@@ -2,7 +2,7 @@
 from outlines.fsm.guide import RegexGuide
 from outlines.fsm.json_schema import build_regex_from_schema
 
-from .common import ensure_numba_compiled, setup_tokenizer  # noqa: E402
+from .common import setup_tokenizer  # noqa: E402
 
 simple_schema = """{
         "$defs": {
@@ -69,7 +69,6 @@ class JsonSchemaBenchmark:
     def setup(self, schema_name):
         self.tokenizer = setup_tokenizer()
         self.schema = schemas[schema_name]
-        ensure_numba_compiled(self.tokenizer)
 
     @cache_disabled()
     def time_json_schema_to_regex(self, schema_name):

benchmarks/bench_numba_compile.py

@@ -1,14 +1,8 @@
 from transformers import AutoTokenizer
 
-from outlines.fsm.guide import RegexGuide
 from outlines.models.transformers import TransformerTokenizer
 
 
 def setup_tokenizer():
     tokenizer = AutoTokenizer.from_pretrained("gpt2")
     return TransformerTokenizer(tokenizer)
-
-
-def ensure_numba_compiled(tokenizer):
-    RegexGuide("a", tokenizer)
-    return True

benchmarks/bench_regex_guide.py

@@ -20,16 +20,16 @@
 import torch
 from lark.indenter import DedentError
 from lark.lexer import UnexpectedCharacters, UnexpectedToken
-
-from outlines import grammars
-from outlines.caching import cache
-from outlines.fsm.parsing import PartialLark, PartialParserState
-from outlines.fsm.regex import (
+from outlines_core.fsm.regex import (
     create_fsm_index_tokenizer,
     make_byte_level_fsm,
     make_deterministic_fsm,
 )
 
+from outlines import grammars
+from outlines.caching import cache
+from outlines.fsm.parsing import PartialLark, PartialParserState
+
 if TYPE_CHECKING:
     from outlines.models.tokenizer import Tokenizer
 

benchmarks/common.py

@@ -34,8 +34,7 @@
 )
 from lark.parsers.lalr_interactive_parser import InteractiveParser
 from lark.parsers.lalr_parser import LALR_Parser, ParseConf, ParserState, _Parser
-
-from outlines.fsm.regex import (
+from outlines_core.fsm.regex import (
     fsm_union,
     get_sub_fsms_from_seq,
     get_token_transition_keys,