[Python-checkins] Fix typing problems reported by mypy in pegen (GH-20297)
Pablo Galindo
webhook-mailer at python.org
Thu May 21 16:39:52 EDT 2020
https://github.com/python/cpython/commit/d10fef35c6ce8a3193b974be7e8c8304b1146153
commit: d10fef35c6ce8a3193b974be7e8c8304b1146153
branch: master
author: Pablo Galindo <Pablogsal at gmail.com>
committer: GitHub <noreply at github.com>
date: 2020-05-21T21:39:44+01:00
summary:
Fix typing problems reported by mypy in pegen (GH-20297)
files:
M Tools/peg_generator/pegen/c_generator.py
M Tools/peg_generator/pegen/parser_generator.py
diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index e8107ec044d0a..586f28cc46975 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -1,5 +1,5 @@
import ast
-from dataclasses import dataclass
+from dataclasses import field, dataclass
import re
from typing import Any, Dict, IO, Optional, List, Text, Tuple, Set
from enum import Enum
@@ -25,6 +25,7 @@
)
from pegen.parser_generator import ParserGenerator
+
EXTENSION_PREFIX = """\
#include "pegen.h"
@@ -63,7 +64,7 @@ class NodeTypes(Enum):
@dataclass
class FunctionCall:
function: str
- arguments: Optional[List[Any]] = None
+ arguments: List[Any] = field(default_factory=list)
assigned_variable: Optional[str] = None
return_type: Optional[str] = None
nodetype: Optional[NodeTypes] = None
@@ -94,7 +95,7 @@ def __init__(
self.gen = parser_generator
self.exact_tokens = exact_tokens
self.non_exact_tokens = non_exact_tokens
- self.cache: Dict[Any, Any] = {}
+ self.cache: Dict[Any, FunctionCall] = {}
self.keyword_cache: Dict[str, int] = {}
def keyword_helper(self, keyword: str) -> FunctionCall:
@@ -171,7 +172,7 @@ def can_we_inline(node: Rhs) -> int:
if node in self.cache:
return self.cache[node]
if can_we_inline(node):
- self.cache[node] = self.visit(node.alts[0].items[0])
+ self.cache[node] = self.generate_call(node.alts[0].items[0])
else:
name = self.gen.name_node(node)
self.cache[node] = FunctionCall(
@@ -183,13 +184,13 @@ def can_we_inline(node: Rhs) -> int:
return self.cache[node]
def visit_NamedItem(self, node: NamedItem) -> FunctionCall:
- call = self.visit(node.item)
+ call = self.generate_call(node.item)
if node.name:
call.assigned_variable = node.name
return call
def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
- call = self.visit(node.node)
+ call = self.generate_call(node.node)
if call.nodetype == NodeTypes.NAME_TOKEN:
return FunctionCall(
function=f"_PyPegen_lookahead_with_name",
@@ -217,7 +218,7 @@ def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall:
return self.lookahead_call_helper(node, 0)
def visit_Opt(self, node: Opt) -> FunctionCall:
- call = self.visit(node.node)
+ call = self.generate_call(node.node)
return FunctionCall(
assigned_variable="_opt_var",
function=call.function,
@@ -266,7 +267,7 @@ def visit_Gather(self, node: Gather) -> FunctionCall:
return self.cache[node]
def visit_Group(self, node: Group) -> FunctionCall:
- return self.visit(node.rhs)
+ return self.generate_call(node.rhs)
def visit_Cut(self, node: Cut) -> FunctionCall:
return FunctionCall(
@@ -276,6 +277,9 @@ def visit_Cut(self, node: Cut) -> FunctionCall:
nodetype=NodeTypes.CUT_OPERATOR,
)
+ def generate_call(self, node: Any) -> FunctionCall:
+ return super().visit(node)
+
class CParserGenerator(ParserGenerator, GrammarVisitor):
def __init__(
@@ -317,17 +321,13 @@ def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None:
self.print(f"goto {goto_target};")
self.print(f"}}")
- def out_of_memory_return(
- self,
- expr: str,
- cleanup_code: Optional[str] = None,
- ) -> None:
+ def out_of_memory_return(self, expr: str, cleanup_code: Optional[str] = None,) -> None:
self.print(f"if ({expr}) {{")
with self.indent():
if cleanup_code is not None:
self.print(cleanup_code)
self.print("p->error_indicator = 1;")
- self.print("PyErr_NoMemory();");
+ self.print("PyErr_NoMemory();")
self.print("return NULL;")
self.print(f"}}")
@@ -484,10 +484,7 @@ def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> N
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
- rhs,
- is_loop=False,
- is_gather=node.is_gather(),
- rulename=node.name,
+ rhs, is_loop=False, is_gather=node.is_gather(), rulename=node.name,
)
if self.debug:
self.print('fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark);')
@@ -518,10 +515,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
- rhs,
- is_loop=True,
- is_gather=node.is_gather(),
- rulename=node.name,
+ rhs, is_loop=True, is_gather=node.is_gather(), rulename=node.name,
)
if is_repeat1:
self.print("if (_n == 0 || p->error_indicator) {")
@@ -567,7 +561,7 @@ def visit_Rule(self, node: Rule) -> None:
self.print("}")
def visit_NamedItem(self, node: NamedItem) -> None:
- call = self.callmakervisitor.visit(node)
+ call = self.callmakervisitor.generate_call(node)
if call.assigned_variable:
call.assigned_variable = self.dedupe(call.assigned_variable)
self.print(call)
@@ -674,7 +668,9 @@ def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) -
self.print("if (_n == _children_capacity) {")
with self.indent():
self.print("_children_capacity *= 2;")
- self.print("void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));")
+ self.print(
+ "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));"
+ )
self.out_of_memory_return(f"!_new_children")
self.print("_children = _new_children;")
self.print("}")
@@ -721,7 +717,7 @@ def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]:
return types
def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]:
- call = self.callmakervisitor.visit(node.item)
+ call = self.callmakervisitor.generate_call(node.item)
name = node.name if node.name else call.assigned_variable
if name is not None:
name = self.dedupe(name)
diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py
index 689022b12da20..364eccb84f479 100644
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -27,7 +27,7 @@ def visit_NameLeaf(self, node: NameLeaf) -> None:
# TODO: Add line/col info to (leaf) nodes
raise GrammarError(f"Dangling reference to rule {node.value!r}")
- def visit_NamedItem(self, node: NameLeaf) -> None:
+ def visit_NamedItem(self, node: NamedItem) -> None:
if node.name and node.name.startswith("_"):
raise GrammarError(f"Variable names cannot start with underscore: '{node.name}'")
self.visit(node.item)
@@ -57,7 +57,7 @@ def __init__(self, grammar: Grammar, tokens: Dict[int, str], file: Optional[IO[T
self.all_rules: Dict[str, Rule] = {} # Rules + temporal rules
self._local_variable_stack: List[List[str]] = []
- def validate_rule_names(self):
+ def validate_rule_names(self) -> None:
for rule in self.rules:
if rule.startswith("_"):
raise GrammarError(f"Rule names cannot start with underscore: '{rule}'")
More information about the Python-checkins
mailing list