gh-109596: Ensure repeated rules in the grammar are not allowed and fix incorrect soft keywords (#109606)

Author: pablogsal

Grammar/python.gram

@@ -19,8 +19,6 @@ _PyPegen_parse(Parser *p)
         result = eval_rule(p);
     } else if (p->start_rule == Py_func_type_input) {
         result = func_type_rule(p);
-    } else if (p->start_rule == Py_fstring_input) {
-        result = fstring_rule(p);
     }
 
     return result;
@@ -89,7 +87,6 @@ file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) }
 interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) }
 eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) }
 func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) }
-fstring[expr_ty]: star_expressions
 
 # GENERAL STATEMENTS
 # ==================
@@ -647,20 +644,20 @@ type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ ['
 
 type_param[type_param_ty] (memo):
     | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) }
-    | '*' a=NAME colon=":" e=expression {
+    | '*' a=NAME colon=':' e=expression {
             RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
                 ? "cannot use constraints with TypeVarTuple"
                 : "cannot use bound with TypeVarTuple")
         }
     | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) }
-    | '**' a=NAME colon=":" e=expression {
+    | '**' a=NAME colon=':' e=expression {
             RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind
                 ? "cannot use constraints with ParamSpec"
                 : "cannot use bound with ParamSpec")
         }
     | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) }
 
-type_param_bound[expr_ty]: ":" e=expression { e }
+type_param_bound[expr_ty]: ':' e=expression { e }
 
 # EXPRESSIONS
 # -----------
@@ -915,7 +912,7 @@ fstring_middle[expr_ty]:
     | fstring_replacement_field
     | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
 fstring_replacement_field[expr_ty]:
-    | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
+    | '{' a=(yield_expr | star_expressions) debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
         _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
     | invalid_replacement_field
 fstring_conversion[ResultTokenWithMetadata*]:

Include/compile.h

@@ -10,9 +10,6 @@ extern "C" {
 #define Py_eval_input 258
 #define Py_func_type_input 345
 
-/* This doesn't need to match anything */
-#define Py_fstring_input 800
-
 #ifndef Py_LIMITED_API
 #  define Py_CPYTHON_COMPILE_H
 #  include "cpython/compile.h"

Lib/test/test_peg_generator/test_pegen.py

@@ -42,6 +42,15 @@ def test_parse_grammar(self) -> None:
         )
         self.assertEqual(repr(rules["term"]), expected_repr)
 
+    def test_repeated_rules(self) -> None:
+        grammar_source = """
+        start: the_rule NEWLINE
+        the_rule: 'b' NEWLINE
+        the_rule: 'a' NEWLINE
+        """
+        with self.assertRaisesRegex(GrammarError, "Repeated rule 'the_rule'"):
+            parse_string(grammar_source, GrammarParser)
+
     def test_long_rule_str(self) -> None:
         grammar_source = """
         start: zero | one | one zero | one one | one zero zero | one zero one | one one zero | one one one

Misc/NEWS.d/next/Core and Builtins/2023-09-20-13-18-08.gh-issue-109596.RG0K2G.rst

@@ -0,0 +1,3 @@
+Fix some tokens in the grammar that were incorrectly marked as soft
+keywords. Also fix some repeated rule names and ensure that repeated rules
+are not allowed. Patch by Pablo Galindo