[Python-3000-checkins] r65777 - in python/branches/py3k: Doc/library/language.rst Doc/library/symtable.rst Include/symtable.h Lib/symtable.py Lib/test/test_symtable.py Modules/symtablemodule.c Python/symtable.c
benjamin.peterson
python-3000-checkins at python.org
Sun Aug 17 20:02:45 CEST 2008
Author: benjamin.peterson
Date: Sun Aug 17 20:02:44 2008
New Revision: 65777
Log:
Merged revisions 65715,65724,65726,65732,65736-65739,65775 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r65715 | benjamin.peterson | 2008-08-16 16:04:16 -0500 (Sat, 16 Aug 2008) | 1 line
add some documentation for symtable
........
r65724 | benjamin.peterson | 2008-08-16 17:11:33 -0500 (Sat, 16 Aug 2008) | 2 lines
include filename and line number in SyntaxError
........
r65726 | georg.brandl | 2008-08-16 17:37:05 -0500 (Sat, 16 Aug 2008) | 2 lines
Review symtable docs.
........
r65732 | benjamin.peterson | 2008-08-16 18:29:40 -0500 (Sat, 16 Aug 2008) | 1 line
PySTEntry's constructor is static; there's no point in a fancy API name
........
r65736 | benjamin.peterson | 2008-08-16 20:09:17 -0500 (Sat, 16 Aug 2008) | 1 line
expose PySTEntry.nested so the symtable module will work
........
r65737 | benjamin.peterson | 2008-08-16 20:17:15 -0500 (Sat, 16 Aug 2008) | 1 line
a few improvements
........
r65738 | benjamin.peterson | 2008-08-16 20:27:30 -0500 (Sat, 16 Aug 2008) | 1 line
fix compile errors
........
r65739 | benjamin.peterson | 2008-08-16 21:23:43 -0500 (Sat, 16 Aug 2008) | 1 line
uhh PySTEntry->ste_unoptimized has to be exposed too
........
r65775 | benjamin.peterson | 2008-08-17 12:13:26 -0500 (Sun, 17 Aug 2008) | 5 lines
get the symtable module back in working order
- Fix broken functions
- Add (hopefully) extensive tests
- Modernize a little
........
Added:
python/branches/py3k/Doc/library/symtable.rst
- copied, changed from r65715, /python/trunk/Doc/library/symtable.rst
Modified:
python/branches/py3k/ (props changed)
python/branches/py3k/Doc/library/language.rst
python/branches/py3k/Include/symtable.h
python/branches/py3k/Lib/symtable.py
python/branches/py3k/Lib/test/test_symtable.py
python/branches/py3k/Modules/symtablemodule.c
python/branches/py3k/Python/symtable.c
Modified: python/branches/py3k/Doc/library/language.rst
==============================================================================
--- python/branches/py3k/Doc/library/language.rst (original)
+++ python/branches/py3k/Doc/library/language.rst Sun Aug 17 20:02:44 2008
@@ -16,6 +16,7 @@
parser.rst
ast.rst
+ symtable.rst
symbol.rst
token.rst
keyword.rst
Copied: python/branches/py3k/Doc/library/symtable.rst (from r65715, /python/trunk/Doc/library/symtable.rst)
==============================================================================
--- /python/trunk/Doc/library/symtable.rst (original)
+++ python/branches/py3k/Doc/library/symtable.rst Sun Aug 17 20:02:44 2008
@@ -1,6 +1,5 @@
-
-:mod:`symtable` -- Access to the compiler's symbol tables
-=========================================================
+:mod:`symtable` --- Access to the compiler's symbol tables
+==========================================================
.. module:: symtable
:synopsis: Interface to the compiler's internal symbol tables.
@@ -20,7 +19,7 @@
.. function:: symtable(code, filename, compile_type)
- Return the toplevel :class:`SymbolTable` for the Python source, *code*.
+ Return the toplevel :class:`SymbolTable` for the Python source *code*.
*filename* is the name of the file containing the code. *compile_type* is
like the *mode* argument to :func:`compile`.
@@ -32,7 +31,6 @@
A namespace table for a block. The constructor is not public.
-
.. method:: get_type()
Return the type of the symbol table. Possible values are ``'class'``,
@@ -46,7 +44,7 @@
Return the table's name. This is the name of the class if the table is
for a class, the name of the function if the table is for a function, or
- ``'top'`` if the table is global.
+ ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``).
.. method:: get_lineno()
@@ -58,7 +56,7 @@
.. method:: is_nested()
- Return ``True`` if the block is nested in another.
+ Return ``True`` if the block is a nested class or function.
.. method:: has_children()
@@ -71,7 +69,7 @@
.. method:: has_import_start()
- Return ``True`` if the block uses a starred import.
+ Return ``True`` if the block uses a starred from-import.
.. method:: get_identifiers()
@@ -95,7 +93,6 @@
A namespace for a function or method. This class inherits
:class:`SymbolTable`.
-
.. method:: get_parameters()
Return a tuple containing names of parameters to this function.
@@ -124,7 +121,7 @@
.. class:: Symbol
- An entry in a :class:`SymbolTable` corresponding to a identifier in the
+ An entry in a :class:`SymbolTable` corresponding to an identifier in the
source. The constructor is not public.
.. method:: get_name()
@@ -153,7 +150,8 @@
.. method:: is_kewordarg()
- Return ``True`` if symbol is a starred arg (receives keywrod arguments).
+ Return ``True`` if the symbol is a two-star arg (receives keyword
+ arguments).
.. method:: is_local()
@@ -161,7 +159,8 @@
.. method:: is_free()
- Return ``True`` if the symbol is used in its block, but not declared.
+ Return ``True`` if the symbol is referenced in its block, but not assigned
+ to.
.. method:: is_assigned()
Modified: python/branches/py3k/Include/symtable.h
==============================================================================
--- python/branches/py3k/Include/symtable.h (original)
+++ python/branches/py3k/Include/symtable.h Sun Aug 17 20:02:44 2008
@@ -36,7 +36,7 @@
PyObject *ste_children; /* list of child blocks */
_Py_block_ty ste_type; /* module, class, or function */
int ste_unoptimized; /* false if namespace is optimized */
- unsigned ste_nested : 1; /* true if block is nested */
+ int ste_nested; /* true if block is nested */
unsigned ste_free : 1; /* true if block has free variables */
unsigned ste_child_free : 1; /* true if a child block has free vars,
including free refs to globals */
Modified: python/branches/py3k/Lib/symtable.py
==============================================================================
--- python/branches/py3k/Lib/symtable.py (original)
+++ python/branches/py3k/Lib/symtable.py Sun Aug 17 20:02:44 2008
@@ -1,10 +1,11 @@
"""Interface to the compiler's internal symbol tables"""
import _symtable
-from _symtable import USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM, \
- DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE, \
- DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND, \
- OPT_IMPORT_STAR
+from _symtable import (USE, DEF_GLOBAL, DEF_LOCAL, DEF_PARAM,
+ DEF_STAR, DEF_DOUBLESTAR, DEF_INTUPLE, DEF_FREE,
+ DEF_FREE_GLOBAL, DEF_FREE_CLASS, DEF_IMPORT, DEF_BOUND,
+ OPT_IMPORT_STAR, SCOPE_OFF, SCOPE_MASK, FREE,
+ GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
import weakref
@@ -38,15 +39,9 @@
newSymbolTable = SymbolTableFactory()
-def is_free(flags):
- if (flags & (USE | DEF_FREE)) \
- and (flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)):
- return True
- if flags & DEF_FREE_CLASS:
- return True
- return False
-class SymbolTable:
+class SymbolTable(object):
+
def __init__(self, raw_table, filename):
self._table = raw_table
self._filename = filename
@@ -59,10 +54,11 @@
kind = "%s " % self.__class__.__name__
if self._table.name == "global":
- return "<%sSymbolTable for module %s>" % (kind, self._filename)
+ return "<{0}SymbolTable for module {1}>".format(kind, self._filename)
else:
- return "<%sSymbolTable for %s in %s>" % (kind, self._table.name,
- self._filename)
+ return "<{0}SymbolTable for {1} in {2}>".format(kind,
+ self._table.name,
+ self._filename)
def get_type(self):
if self._table.type == _symtable.TYPE_MODULE:
@@ -72,7 +68,7 @@
if self._table.type == _symtable.TYPE_CLASS:
return "class"
assert self._table.type in (1, 2, 3), \
- "unexpected type: %s" % self._table.type
+ "unexpected type: {0}".format(self._table.type)
def get_id(self):
return self._table.id
@@ -124,6 +120,7 @@
return [newSymbolTable(st, self._filename)
for st in self._table.children]
+
class Function(SymbolTable):
# Default values for instance variables
@@ -148,15 +145,18 @@
def get_globals(self):
if self.__globals is None:
- glob = DEF_GLOBAL | DEF_FREE_GLOBAL
- self.__globals = self.__idents_matching(lambda x:x & glob)
+ glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT)
+ test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob
+ self.__globals = self.__idents_matching(test)
return self.__globals
def get_frees(self):
if self.__frees is None:
+ is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE
self.__frees = self.__idents_matching(is_free)
return self.__frees
+
class Class(SymbolTable):
__methods = None
@@ -169,14 +169,17 @@
self.__methods = tuple(d)
return self.__methods
-class Symbol:
+
+class Symbol(object):
+
def __init__(self, name, flags, namespaces=None):
self.__name = name
self.__flags = flags
+ self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope()
self.__namespaces = namespaces or ()
def __repr__(self):
- return "<symbol '%s'>" % self.__name
+ return "<symbol {0!r}>".format(self.__name)
def get_name(self):
return self.__name
@@ -188,8 +191,7 @@
return bool(self.__flags & DEF_PARAM)
def is_global(self):
- return bool((self.__flags & DEF_GLOBAL)
- or (self.__flags & DEF_FREE_GLOBAL))
+ return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT))
def is_vararg(self):
return bool(self.__flags & DEF_STAR)
@@ -201,12 +203,7 @@
return bool(self.__flags & DEF_BOUND)
def is_free(self):
- if (self.__flags & (USE | DEF_FREE)) \
- and (self.__flags & (DEF_LOCAL | DEF_PARAM | DEF_GLOBAL)):
- return True
- if self.__flags & DEF_FREE_CLASS:
- return True
- return False
+ return bool(self.__scope == FREE)
def is_imported(self):
return bool(self.__flags & DEF_IMPORT)
Modified: python/branches/py3k/Lib/test/test_symtable.py
==============================================================================
--- python/branches/py3k/Lib/test/test_symtable.py (original)
+++ python/branches/py3k/Lib/test/test_symtable.py Sun Aug 17 20:02:44 2008
@@ -1,31 +1,161 @@
-from test import support
-
+"""
+Test the API of the symtable module.
+"""
import symtable
import unittest
+from test import support
+
+
+TEST_CODE = """
+import sys
-## XXX
-## Test disabled because symtable module needs to be rewritten for new compiler
+glob = 42
-##vereq(symbols[0].name, "global")
-##vereq(len([ste for ste in symbols.values() if ste.name == "f"]), 1)
+class Mine:
+ instance_var = 24
+ def a_method(p1, p2):
+ pass
+
+def spam(a, b, *var, **kw):
+ global bar
+ bar = 47
+ x = 23
+ glob
+ def internal():
+ return x
+ return internal
+
+def foo():
+ pass
+
+def namespace_test(): pass
+def namespace_test(): pass
+"""
+
+
+def find_block(block, name):
+ for ch in block.get_children():
+ if ch.get_name() == name:
+ return ch
-### Bug tickler: SyntaxError file name correct whether error raised
-### while parsing or building symbol table.
-##def checkfilename(brokencode):
-## try:
-## _symtable.symtable(brokencode, "spam", "exec")
-## except SyntaxError, e:
-## vereq(e.filename, "spam")
-## else:
-## raise TestFailed("no SyntaxError for %r" % (brokencode,))
-##checkfilename("def f(x): foo)(") # parse-time
-##checkfilename("def f(x): global x") # symtable-build-time
class SymtableTest(unittest.TestCase):
- def test_invalid_args(self):
- self.assertRaises(TypeError, symtable.symtable, "42")
- self.assertRaises(ValueError, symtable.symtable, "42", "?", "")
+
+ top = symtable.symtable(TEST_CODE, "?", "exec")
+ # These correspond to scopes in TEST_CODE
+ Mine = find_block(top, "Mine")
+ a_method = find_block(Mine, "a_method")
+ spam = find_block(top, "spam")
+ internal = find_block(spam, "internal")
+ foo = find_block(top, "foo")
+
+ def test_type(self):
+ self.assertEqual(self.top.get_type(), "module")
+ self.assertEqual(self.Mine.get_type(), "class")
+ self.assertEqual(self.a_method.get_type(), "function")
+ self.assertEqual(self.spam.get_type(), "function")
+ self.assertEqual(self.internal.get_type(), "function")
+
+ def test_optimized(self):
+ self.assertFalse(self.top.is_optimized())
+ self.assertFalse(self.top.has_exec())
+
+ self.assertTrue(self.spam.is_optimized())
+
+ def test_nested(self):
+ self.assertFalse(self.top.is_nested())
+ self.assertFalse(self.Mine.is_nested())
+ self.assertFalse(self.spam.is_nested())
+ self.assertTrue(self.internal.is_nested())
+
+ def test_children(self):
+ self.assertTrue(self.top.has_children())
+ self.assertTrue(self.Mine.has_children())
+ self.assertFalse(self.foo.has_children())
+
+ def test_lineno(self):
+ self.assertEqual(self.top.get_lineno(), 0)
+ self.assertEqual(self.spam.get_lineno(), 11)
+
+ def test_function_info(self):
+ func = self.spam
+ self.assertEqual(func.get_parameters(), ("a", "b", "kw", "var"))
+ self.assertEqual(func.get_locals(),
+ ("a", "b", "bar", "glob", "internal", "kw", "var", "x"))
+ self.assertEqual(func.get_globals(), ("bar", "glob"))
+ self.assertEqual(self.internal.get_frees(), ("x",))
+
+ def test_globals(self):
+ self.assertTrue(self.spam.lookup("glob").is_global())
+ self.assertTrue(self.spam.lookup("bar").is_global())
+ self.assertFalse(self.internal.lookup("x").is_global())
+ self.assertFalse(self.Mine.lookup("instance_var").is_global())
+
+ def test_local(self):
+ self.assertTrue(self.spam.lookup("x").is_local())
+ self.assertFalse(self.internal.lookup("x").is_local())
+
+ def test_referenced(self):
+ self.assertTrue(self.internal.lookup("x").is_referenced())
+ self.assertTrue(self.spam.lookup("internal").is_referenced())
+ self.assertFalse(self.spam.lookup("x").is_referenced())
+
+ def test_parameters(self):
+ for sym in ("a", "var", "kw"):
+ self.assertTrue(self.spam.lookup(sym).is_parameter())
+ self.assertFalse(self.spam.lookup("x").is_parameter())
+
+ def test_symbol_lookup(self):
+ self.assertEqual(len(self.top.get_identifiers()),
+ len(self.top.get_symbols()))
+
+ self.assertRaises(KeyError, self.top.lookup, "not_here")
+
+ def test_namespaces(self):
+ self.assertTrue(self.top.lookup("Mine").is_namespace())
+ self.assertTrue(self.Mine.lookup("a_method").is_namespace())
+ self.assertTrue(self.top.lookup("spam").is_namespace())
+ self.assertTrue(self.spam.lookup("internal").is_namespace())
+ self.assertTrue(self.top.lookup("namespace_test").is_namespace())
+ self.assertFalse(self.spam.lookup("x").is_namespace())
+
+ self.assert_(self.top.lookup("spam").get_namespace() is self.spam)
+ ns_test = self.top.lookup("namespace_test")
+ self.assertEqual(len(ns_test.get_namespaces()), 2)
+ self.assertRaises(ValueError, ns_test.get_namespace)
+
+ def test_assigned(self):
+ self.assertTrue(self.spam.lookup("x").is_assigned())
+ self.assertTrue(self.spam.lookup("bar").is_assigned())
+ self.assertTrue(self.top.lookup("spam").is_assigned())
+ self.assertTrue(self.Mine.lookup("a_method").is_assigned())
+ self.assertFalse(self.internal.lookup("x").is_assigned())
+
+ def test_imported(self):
+ self.assertTrue(self.top.lookup("sys").is_imported())
+
+ def test_name(self):
+ self.assertEqual(self.top.get_name(), "top")
+ self.assertEqual(self.spam.get_name(), "spam")
+ self.assertEqual(self.spam.lookup("x").get_name(), "x")
+ self.assertEqual(self.Mine.get_name(), "Mine")
+
+ def test_class_info(self):
+ self.assertEqual(self.Mine.get_methods(), ('a_method',))
+
+ def test_filename_correct(self):
+ ### Bug tickler: SyntaxError file name correct whether error raised
+ ### while parsing or building symbol table.
+ def checkfilename(brokencode):
+ try:
+ symtable.symtable(brokencode, "spam", "exec")
+ except SyntaxError as e:
+ self.assertEqual(e.filename, "spam")
+ else:
+ self.fail("no SyntaxError for %r" % (brokencode,))
+ checkfilename("def f(x): foo)(") # parse-time
+ checkfilename("def f(x): global x") # symtable-build-time
def test_eval(self):
symbols = symtable.symtable("42", "?", "eval")
Modified: python/branches/py3k/Modules/symtablemodule.c
==============================================================================
--- python/branches/py3k/Modules/symtablemodule.c (original)
+++ python/branches/py3k/Modules/symtablemodule.c Sun Aug 17 20:02:44 2008
@@ -92,6 +92,10 @@
PyModule_AddIntConstant(m, "GLOBAL_IMPLICIT", GLOBAL_IMPLICIT);
PyModule_AddIntConstant(m, "FREE", FREE);
PyModule_AddIntConstant(m, "CELL", CELL);
+
+ PyModule_AddIntConstant(m, "SCOPE_OFF", SCOPE_OFFSET);
+ PyModule_AddIntConstant(m, "SCOPE_MASK", SCOPE_MASK);
+
if (PyErr_Occurred()) {
Py_DECREF(m);
m = 0;
Modified: python/branches/py3k/Python/symtable.c
==============================================================================
--- python/branches/py3k/Python/symtable.c (original)
+++ python/branches/py3k/Python/symtable.c Sun Aug 17 20:02:44 2008
@@ -22,9 +22,9 @@
#define RETURN_VAL_IN_GENERATOR \
"'return' with argument inside generator"
-/* XXX(nnorwitz): change name since static? */
+
static PySTEntryObject *
-PySTEntry_New(struct symtable *st, identifier name, _Py_block_ty block,
+ste_new(struct symtable *st, identifier name, _Py_block_ty block,
void *key, int lineno)
{
PySTEntryObject *ste = NULL;
@@ -114,6 +114,8 @@
{"symbols", T_OBJECT, OFF(ste_symbols), READONLY},
{"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
{"children", T_OBJECT, OFF(ste_children), READONLY},
+ {"optimized",T_INT, OFF(ste_unoptimized), READONLY},
+ {"nested", T_INT, OFF(ste_nested), READONLY},
{"type", T_INT, OFF(ste_type), READONLY},
{"lineno", T_INT, OFF(ste_lineno), READONLY},
{NULL}
@@ -388,6 +390,9 @@
PyErr_Format(PyExc_SyntaxError,
"name '%U' is parameter and global",
name);
+ PyErr_SyntaxLocation(ste->ste_table->st_filename,
+ ste->ste_lineno);
+
return 0;
}
if (flags & DEF_NONLOCAL) {
@@ -788,7 +793,7 @@
return 1;
}
-/* symtable_enter_block() gets a reference via PySTEntry_New().
+/* symtable_enter_block() gets a reference via ste_new.
This reference is released when the block is exited, via the DECREF
in symtable_exit_block().
*/
@@ -825,7 +830,7 @@
}
Py_DECREF(st->st_cur);
}
- st->st_cur = PySTEntry_New(st, name, block, ast, lineno);
+ st->st_cur = ste_new(st, name, block, ast, lineno);
if (st->st_cur == NULL)
return 0;
if (name == GET_IDENTIFIER(top))
More information about the Python-3000-checkins
mailing list