From 336317897b22206ae3f9b4d92027f406ddcf79e4 Mon Sep 17 00:00:00 2001
From: Pablo Galindo <pablogsal@gmail.com>
Date: Wed, 16 Oct 2024 13:51:12 +0100
Subject: [PATCH] gh-125588: Teach the python PEG generator the new f-string
tokens
Signed-off-by: Pablo Galindo <pablogsal@gmail.com>
---
Lib/test/test_peg_generator/test_pegen.py | 8 +++++
...-10-16-13-52-48.gh-issue-125588.kCahyO.rst | 2 ++
Tools/peg_generator/pegen/grammar_parser.py | 17 ++++++++++-
Tools/peg_generator/pegen/metagrammar.gram | 3 ++
Tools/peg_generator/pegen/parser.py | 30 +++++++++++++++++++
Tools/peg_generator/pegen/python_generator.py | 3 +-
6 files changed, 61 insertions(+), 2 deletions(-)
create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst
diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py
index 86db767b99a228..081088a80bad33 100644
--- a/Lib/test/test_peg_generator/test_pegen.py
+++ b/Lib/test/test_peg_generator/test_pegen.py
@@ -506,6 +506,14 @@ def test_python_expr(self) -> None:
val = eval(code)
self.assertEqual(val, 3.0)
+ def test_f_string_in_action(self) -> None:
+ grammar = """
+ start: n=NAME NEWLINE? $ { f"name -> {n.string}" }
+ """
+ parser_class = make_parser(grammar)
+ node = parse_string("a", parser_class)
+ self.assertEqual(node.strip(), "name -> a")
+
def test_nullable(self) -> None:
grammar_source = """
start: sign NUMBER
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst
new file mode 100644
index 00000000000000..1d59a9c3c205b8
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-16-13-52-48.gh-issue-125588.kCahyO.rst
@@ -0,0 +1,2 @@
+The Python PEG generator can now use f-strings in the grammar actions. Patch
+by Pablo Galindo
diff --git a/Tools/peg_generator/pegen/grammar_parser.py b/Tools/peg_generator/pegen/grammar_parser.py
index bf31fe53263645..2e3a607f7209b0 100644
--- a/Tools/peg_generator/pegen/grammar_parser.py
+++ b/Tools/peg_generator/pegen/grammar_parser.py
@@ -575,7 +575,7 @@ def target_atoms(self) -> Optional[str]:
@memoize
def target_atom(self) -> Optional[str]:
- # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | "?" | ":" | !"}" !"]" OP
+ # target_atom: "{" ~ target_atoms? "}" | "[" ~ target_atoms? "]" | NAME "*" | NAME | NUMBER | STRING | FSTRING_START | FSTRING_MIDDLE | FSTRING_END | "?" | ":" | !"}" !"]" OP
mark = self._mark()
cut = False
if (
@@ -625,6 +625,21 @@ def target_atom(self) -> Optional[str]:
):
return string . string
self._reset(mark)
+ if (
+ (fstring_start := self.fstring_start())
+ ):
+ return fstring_start . string
+ self._reset(mark)
+ if (
+ (fstring_middle := self.fstring_middle())
+ ):
+ return fstring_middle . string
+ self._reset(mark)
+ if (
+ (fstring_end := self.fstring_end())
+ ):
+ return fstring_end . string
+ self._reset(mark)
if (
(literal := self.expect("?"))
):
diff --git a/Tools/peg_generator/pegen/metagrammar.gram b/Tools/peg_generator/pegen/metagrammar.gram
index f22c334ca371b2..f484c4781823bc 100644
--- a/Tools/peg_generator/pegen/metagrammar.gram
+++ b/Tools/peg_generator/pegen/metagrammar.gram
@@ -126,6 +126,9 @@ target_atom[str]:
| NAME { name.string }
| NUMBER { number.string }
| STRING { string.string }
+ | FSTRING_START { fstring_start.string }
+ | FSTRING_MIDDLE { fstring_middle.string }
+ | FSTRING_END { fstring_end.string }
| "?" { "?" }
| ":" { ":" }
| !"}" !"]" OP { op.string }
diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py
index ed0aec9db2443f..692eb9ed2417d7 100644
--- a/Tools/peg_generator/pegen/parser.py
+++ b/Tools/peg_generator/pegen/parser.py
@@ -205,6 +205,36 @@ def string(self) -> Optional[tokenize.TokenInfo]:
return self._tokenizer.getnext()
return None
+ @memoize
+ def fstring_start(self) -> Optional[tokenize.TokenInfo]:
+ FSTRING_START = getattr(token, "FSTRING_START")
+ if not FSTRING_START:
+ return None
+ tok = self._tokenizer.peek()
+ if tok.type == FSTRING_START:
+ return self._tokenizer.getnext()
+ return None
+
+ @memoize
+ def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
+ FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE")
+ if not FSTRING_MIDDLE:
+ return None
+ tok = self._tokenizer.peek()
+ if tok.type == FSTRING_MIDDLE:
+ return self._tokenizer.getnext()
+ return None
+
+ @memoize
+ def fstring_end(self) -> Optional[tokenize.TokenInfo]:
+ FSTRING_END = getattr(token, "FSTRING_END")
+ if not FSTRING_END:
+ return None
+ tok = self._tokenizer.peek()
+ if tok.type == FSTRING_END:
+ return self._tokenizer.getnext()
+ return None
+
@memoize
def op(self) -> Optional[tokenize.TokenInfo]:
tok = self._tokenizer.peek()
diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py
index 588d3d3f6ef8f8..6bc34b52282c12 100644
--- a/Tools/peg_generator/pegen/python_generator.py
+++ b/Tools/peg_generator/pegen/python_generator.py
@@ -99,7 +99,8 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]:
name = node.value
if name == "SOFT_KEYWORD":
return "soft_keyword", "self.soft_keyword()"
- if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT"):
+ if name in ("NAME", "NUMBER", "STRING", "OP", "TYPE_COMMENT",
+ "FSTRING_END", "FSTRING_MIDDLE", "FSTRING_START"):
name = name.lower()
return name, f"self.{name}()"
if name in ("NEWLINE", "DEDENT", "INDENT", "ENDMARKER"):