gh-125588: Teach the python PEG generator the new f-string tokens

Author: pablogsal

Lib/test/test_peg_generator/test_pegen.py

@@ -506,6 +506,14 @@ def test_python_expr(self) -> None:
         val = eval(code)
         self.assertEqual(val, 3.0)
 
+    def test_f_string_in_action(self) -> None:
+        grammar = """
+        start: n=NAME NEWLINE? $ { f"name -> {n.string}" }
+        """
+        parser_class = make_parser(grammar)
+        node = parse_string("a", parser_class)
+        self.assertEqual(node.strip(), "name ->  a")
+
     def test_nullable(self) -> None:
         grammar_source = """
         start: sign NUMBER

Tools/peg_generator/pegen/grammar_parser.py

@@ -126,6 +126,9 @@ target_atom[str]:
     | NAME { name.string }
     | NUMBER { number.string }
     | STRING { string.string }
+    | FSTRING_START { fstring_start.string }
+    | FSTRING_MIDDLE { fstring_middle.string }
+    | FSTRING_END { fstring_end.string }
     | "?" { "?" }
     | ":" { ":" }
     | !"}" !"]" OP { op.string }

Tools/peg_generator/pegen/metagrammar.gram

@@ -205,6 +205,27 @@ def string(self) -> Optional[tokenize.TokenInfo]:
             return self._tokenizer.getnext()
         return None
 
+    @memoize
+    def fstring_start(self) -> Optional[tokenize.TokenInfo]:
+        tok = self._tokenizer.peek()
+        if tok.type == token.FSTRING_START:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
+    def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
+        tok = self._tokenizer.peek()
+        if tok.type == token.FSTRING_MIDDLE:
+            return self._tokenizer.getnext()
+        return None
+
+    @memoize
+    def fstring_end(self) -> Optional[tokenize.TokenInfo]:
+        tok = self._tokenizer.peek()
+        if tok.type == token.FSTRING_END:
+            return self._tokenizer.getnext()
+        return None
+
     @memoize
     def op(self) -> Optional[tokenize.TokenInfo]:
         tok = self._tokenizer.peek()

Tools/peg_generator/pegen/parser.py

@@ -99,7 +99,8 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]:
         name = node.value
         if name == "SOFT_KEYWORD":
             return "soft_keyword", "self.soft_keyword()"
-        if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"):
+        if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT",
+            "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"):
             name = name.lower()
             return name, f"self.{name}()"
         if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):