[Python-checkins] python/dist/src/Lib/compiler ast.py, 1.22,
1.23 pycodegen.py, 1.66, 1.67 symbols.py, 1.13,
1.14 transformer.py, 1.38, 1.39
rhettinger at users.sourceforge.net
rhettinger at users.sourceforge.net
Wed May 19 04:20:42 EDT 2004
- Previous message: [Python-checkins] python/dist/src/Lib symbol.py,1.15,1.16
- Next message: [Python-checkins] python/dist/src/Lib/test test_genexps.py, NONE,
1.1 test_deque.py, 1.10, 1.11 test_grammar.py, 1.49,
1.50 test_parser.py, 1.17, 1.18
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Update of /cvsroot/python/python/dist/src/Lib/compiler
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv9770/Lib/compiler
Modified Files:
ast.py pycodegen.py symbols.py transformer.py
Log Message:
SF patch #872326: Generator expression implementation
(Code contributed by Jiwon Seo.)
The documentation portion of the patch is being re-worked and will be
checked-in soon. Likewise, PEP 289 will be updated to reflect Guido's
rationale for the design decisions on binding behavior (as described in
in his patch comments and in discussions on python-dev).
The test file, test_genexps.py, is written in doctest format and is
meant to exercise all aspects of the the patch. Further additions are
welcome from everyone. Please stress test this new feature as much as
possible before the alpha release.
Index: ast.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/compiler/ast.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -C2 -d -r1.22 -r1.23
*** ast.py 11 Aug 2003 16:20:39 -0000 1.22
--- ast.py 19 May 2004 08:20:06 -0000 1.23
***************
*** 1237,1240 ****
--- 1237,1316 ----
return "ListCompFor(%s, %s, %s)" % (repr(self.assign), repr(self.list), repr(self.ifs))
+ class GenExpr(Node):
+ nodes["genexpr"] = "GenExpr"
+ def __init__(self, code):
+ self.code = code
+ self.argnames = ['[outmost-iterable]']
+ self.varargs = self.kwargs = None
+
+ def getChildren(self):
+ return self.code,
+
+ def getChildNodes(self):
+ return self.code,
+
+ def __repr__(self):
+ return "GenExpr(%s)" % (repr(self.code),)
+
+ class GenExprInner(Node):
+ nodes["genexprinner"] = "GenExprInner"
+ def __init__(self, expr, quals):
+ self.expr = expr
+ self.quals = quals
+
+ def getChildren(self):
+ children = []
+ children.append(self.expr)
+ children.extend(flatten(self.quals))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.expr)
+ nodelist.extend(flatten_nodes(self.quals))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenExprInner(%s, %s)" % (repr(self.expr), repr(self.quals))
+
+ class GenExprFor(Node):
+ nodes["genexprfor"] = "GenExprFor"
+ def __init__(self, assign, iter, ifs):
+ self.assign = assign
+ self.iter = iter
+ self.ifs = ifs
+ self.is_outmost = False
+
+ def getChildren(self):
+ children = []
+ children.append(self.assign)
+ children.append(self.iter)
+ children.extend(flatten(self.ifs))
+ return tuple(children)
+
+ def getChildNodes(self):
+ nodelist = []
+ nodelist.append(self.assign)
+ nodelist.append(self.iter)
+ nodelist.extend(flatten_nodes(self.ifs))
+ return tuple(nodelist)
+
+ def __repr__(self):
+ return "GenExprFor(%s, %s, %s)" % (repr(self.assign), repr(self.iter), repr(self.ifs))
+
+ class GenExprIf(Node):
+ nodes["genexprif"] = "GenExprIf"
+ def __init__(self, test):
+ self.test = test
+
+ def getChildren(self):
+ return self.test,
+
+ def getChildNodes(self):
+ return self.test,
+
+ def __repr__(self):
+ return "GenExprIf(%s)" % (repr(self.test),)
+
klasses = globals()
for k in nodes.keys():
Index: pycodegen.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/compiler/pycodegen.py,v
retrieving revision 1.66
retrieving revision 1.67
diff -C2 -d -r1.66 -r1.67
*** pycodegen.py 21 Mar 2004 15:18:50 -0000 1.66
--- pycodegen.py 19 May 2004 08:20:07 -0000 1.67
***************
*** 620,623 ****
--- 620,696 ----
self.emit('POP_TOP')
+ def visitGenExpr(self, node):
+ gen = GenExprCodeGenerator(node, self.scopes, self.class_name,
+ self.get_module())
+ walk(node.code, gen)
+ gen.finish()
+ self.set_lineno(node)
+ frees = gen.scope.get_free_vars()
+ if frees:
+ for name in frees:
+ self.emit('LOAD_CLOSURE', name)
+ self.emit('LOAD_CONST', gen)
+ self.emit('MAKE_CLOSURE', 0)
+ else:
+ self.emit('LOAD_CONST', gen)
+ self.emit('MAKE_FUNCTION', 0)
+
+ # precomputation of outmost iterable
+ self.visit(node.code.quals[0].iter)
+ self.emit('GET_ITER')
+ self.emit('CALL_FUNCTION', 1)
+
+ def visitGenExprInner(self, node):
+ self.set_lineno(node)
+ # setup list
+
+ stack = []
+ for i, for_ in zip(range(len(node.quals)), node.quals):
+ start, anchor = self.visit(for_)
+ cont = None
+ for if_ in for_.ifs:
+ if cont is None:
+ cont = self.newBlock()
+ self.visit(if_, cont)
+ stack.insert(0, (start, cont, anchor))
+
+ self.visit(node.expr)
+ self.emit('YIELD_VALUE')
+
+ for start, cont, anchor in stack:
+ if cont:
+ skip_one = self.newBlock()
+ self.emit('JUMP_FORWARD', skip_one)
+ self.startBlock(cont)
+ self.emit('POP_TOP')
+ self.nextBlock(skip_one)
+ self.emit('JUMP_ABSOLUTE', start)
+ self.startBlock(anchor)
+ self.emit('LOAD_CONST', None)
+
+ def visitGenExprFor(self, node):
+ start = self.newBlock()
+ anchor = self.newBlock()
+
+ if node.is_outmost:
+ self.loadName('[outmost-iterable]')
+ else:
+ self.visit(node.iter)
+ self.emit('GET_ITER')
+
+ self.nextBlock(start)
+ self.set_lineno(node, force=True)
+ self.emit('FOR_ITER', anchor)
+ self.nextBlock()
+ self.visit(node.assign)
+ return start, anchor
+
+ def visitGenExprIf(self, node, branch):
+ self.set_lineno(node, force=True)
+ self.visit(node.test)
+ self.emit('JUMP_IF_FALSE', branch)
+ self.newBlock()
+ self.emit('POP_TOP')
+
# exception related
***************
*** 1200,1203 ****
--- 1273,1277 ----
else:
name = func.name
+
args, hasTupleArg = generateArgList(func.argnames)
self.graph = pyassem.PyFlowGraph(name, func.filename, args,
***************
*** 1264,1267 ****
--- 1338,1356 ----
self.graph.setFlag(CO_GENERATOR)
+ class GenExprCodeGenerator(NestedScopeMixin, AbstractFunctionCode,
+ CodeGenerator):
+ super_init = CodeGenerator.__init__ # call be other init
+ scopes = None
+
+ __super_init = AbstractFunctionCode.__init__
+
+ def __init__(self, gexp, scopes, class_name, mod):
+ self.scopes = scopes
+ self.scope = scopes[gexp]
+ self.__super_init(gexp, scopes, 1, class_name, mod)
+ self.graph.setFreeVars(self.scope.get_free_vars())
+ self.graph.setCellVars(self.scope.get_cell_vars())
+ self.graph.setFlag(CO_GENERATOR)
+
class AbstractClassCode:
Index: symbols.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/compiler/symbols.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -C2 -d -r1.13 -r1.14
*** symbols.py 28 Aug 2003 02:09:26 -0000 1.13
--- symbols.py 19 May 2004 08:20:08 -0000 1.14
***************
*** 180,183 ****
--- 180,198 ----
pass
+ class GenExprScope(Scope):
+ __super_init = Scope.__init__
+
+ __counter = 1
+
+ def __init__(self, module, klass=None):
+ i = self.__counter
+ self.__counter += 1
+ self.__super_init("generator expression<%d>"%i, module, klass)
+ self.add_param('[outmost-iterable]')
+
+ def get_names(self):
+ keys = Scope.get_names()
+ return keys
+
class LambdaScope(FunctionScope):
__super_init = Scope.__init__
***************
*** 221,224 ****
--- 236,265 ----
self.handle_free_vars(scope, parent)
+ def visitGenExpr(self, node, parent):
+ scope = GenExprScope(self.module, self.klass);
+ if parent.nested or isinstance(parent, FunctionScope) \
+ or isinstance(parent, GenExprScope):
+ scope.nested = 1
+
+ self.scopes[node] = scope
+ self.visit(node.code, scope)
+
+ self.handle_free_vars(scope, parent)
+
+ def visitGenExprInner(self, node, scope):
+ for genfor in node.quals:
+ self.visit(genfor, scope)
+
+ self.visit(node.expr, scope)
+
+ def visitGenExprFor(self, node, scope):
+ self.visit(node.assign, scope, 1)
+ self.visit(node.iter, scope)
+ for if_ in node.ifs:
+ self.visit(if_, scope)
+
+ def visitGenExprIf(self, node, scope):
+ self.visit(node.test, scope)
+
def visitLambda(self, node, parent, assign=0):
# Lambda is an expression, so it could appear in an expression
Index: transformer.py
===================================================================
RCS file: /cvsroot/python/python/dist/src/Lib/compiler/transformer.py,v
retrieving revision 1.38
retrieving revision 1.39
diff -C2 -d -r1.38 -r1.39
*** transformer.py 6 Apr 2003 09:00:45 -0000 1.38
--- transformer.py 19 May 2004 08:20:08 -0000 1.39
***************
*** 535,538 ****
--- 535,544 ----
exprlist = testlist
+ def testlist_gexp(self, nodelist):
+ if len(nodelist) == 2 and nodelist[1][0] == symbol.gen_for:
+ test = self.com_node(nodelist[0])
+ return self.com_generator_expression(test, nodelist[1])
+ return self.testlist(nodelist)
+
def test(self, nodelist):
# and_test ('or' and_test)* | lambdef
***************
*** 1086,1089 ****
--- 1092,1137 ----
return List(values)
+ if hasattr(symbol, 'gen_for'):
+ def com_generator_expression(self, expr, node):
+ # gen_iter: gen_for | gen_if
+ # gen_for: 'for' exprlist 'in' test [gen_iter]
+ # gen_if: 'if' test [gen_iter]
+
+ lineno = node[1][2]
+ fors = []
+ while node:
+ t = node[1][1]
+ if t == 'for':
+ assignNode = self.com_assign(node[2], OP_ASSIGN)
+ genNode = self.com_node(node[4])
+ newfor = GenExprFor(assignNode, genNode, [])
+ newfor.lineno = node[1][2]
+ fors.append(newfor)
+ if (len(node)) == 5:
+ node = None
+ else:
+ node = self.com_gen_iter(node[5])
+ elif t == 'if':
+ test = self.com_node(node[2])
+ newif = GenExprIf(test)
+ newif.lineno = node[1][2]
+ newfor.ifs.append(newif)
+ if len(node) == 3:
+ node = None
+ else:
+ node = self.com_gen_iter(node[3])
+ else:
+ raise SyntaxError, \
+ ("unexpected generator expression element: %s %d"
+ % (node, lineno))
+ fors[0].is_outmost = True
+ n = GenExpr(GenExprInner(expr, fors))
+ n.lineno = lineno
+ return n
+
+ def com_gen_iter(self, node):
+ assert node[0] == symbol.gen_iter
+ return node[1]
+
def com_dictmaker(self, nodelist):
# dictmaker: test ':' test (',' test ':' value)* [',']
***************
*** 1123,1126 ****
--- 1171,1176 ----
break
kw, result = self.com_argument(node, kw)
+ if len_nodelist != 2 and isinstance(result, GenExpr):
+ raise SyntaxError, 'generator expression needs parenthesis'
args.append(result)
else:
***************
*** 1149,1152 ****
--- 1199,1205 ----
def com_argument(self, nodelist, kw):
+ if len(nodelist) == 3 and nodelist[2][0] == symbol.gen_for:
+ test = self.com_node(nodelist[1])
+ return 0, self.com_generator_expression(test, nodelist[2])
if len(nodelist) == 2:
if kw:
- Previous message: [Python-checkins] python/dist/src/Lib symbol.py,1.15,1.16
- Next message: [Python-checkins] python/dist/src/Lib/test test_genexps.py, NONE,
1.1 test_deque.py, 1.10, 1.11 test_grammar.py, 1.49,
1.50 test_parser.py, 1.17, 1.18
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the Python-checkins
mailing list