[Python-checkins] cpython: create NameConstant AST class for None, True, and False literals (closes #16619)
benjamin.peterson
python-checkins at python.org
Thu Dec 6 23:41:13 CET 2012
http://hg.python.org/cpython/rev/03f92a9f0875
changeset: 80738:03f92a9f0875
parent: 80736:a58204570a7c
user: Benjamin Peterson <benjamin at python.org>
date: Thu Dec 06 17:41:04 2012 -0500
summary:
create NameConstant AST class for None, True, and False literals (closes #16619)
files:
Include/Python-ast.h | 12 +++-
Include/asdl.h | 1 +
Lib/ast.py | 6 +-
Lib/test/test_ast.py | 13 +++-
Lib/test/test_syntax.py | 4 +-
Misc/NEWS | 4 +
Parser/Python.asdl | 3 +-
Parser/asdl.py | 2 +-
Parser/asdl_c.py | 12 ++++
Python/Python-ast.c | 73 +++++++++++++++++++++++++++++
Python/ast.c | 22 +++++++-
Python/compile.c | 21 +++++--
Python/peephole.c | 46 ------------------
Python/symtable.c | 1 +
Tools/parser/unparse.py | 3 +
15 files changed, 152 insertions(+), 71 deletions(-)
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
--- a/Include/Python-ast.h
+++ b/Include/Python-ast.h
@@ -182,8 +182,9 @@
SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
Yield_kind=12, YieldFrom_kind=13, Compare_kind=14,
Call_kind=15, Num_kind=16, Str_kind=17, Bytes_kind=18,
- Ellipsis_kind=19, Attribute_kind=20, Subscript_kind=21,
- Starred_kind=22, Name_kind=23, List_kind=24, Tuple_kind=25};
+ NameConstant_kind=19, Ellipsis_kind=20, Attribute_kind=21,
+ Subscript_kind=22, Starred_kind=23, Name_kind=24,
+ List_kind=25, Tuple_kind=26};
struct _expr {
enum _expr_kind kind;
union {
@@ -279,6 +280,10 @@
} Bytes;
struct {
+ singleton value;
+ } NameConstant;
+
+ struct {
expr_ty value;
identifier attr;
expr_context_ty ctx;
@@ -509,6 +514,9 @@
expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena);
#define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3)
expr_ty _Py_Bytes(bytes s, int lineno, int col_offset, PyArena *arena);
+#define NameConstant(a0, a1, a2, a3) _Py_NameConstant(a0, a1, a2, a3)
+expr_ty _Py_NameConstant(singleton value, int lineno, int col_offset, PyArena
+ *arena);
#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2)
expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena);
#define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5)
diff --git a/Include/asdl.h b/Include/asdl.h
--- a/Include/asdl.h
+++ b/Include/asdl.h
@@ -5,6 +5,7 @@
typedef PyObject * string;
typedef PyObject * bytes;
typedef PyObject * object;
+typedef PyObject * singleton;
/* It would be nice if the code generated by asdl_c.py was completely
independent of Python, but it is a goal the requires too much work
diff --git a/Lib/ast.py b/Lib/ast.py
--- a/Lib/ast.py
+++ b/Lib/ast.py
@@ -42,7 +42,6 @@
Python literal structures: strings, bytes, numbers, tuples, lists, dicts,
sets, booleans, and None.
"""
- _safe_names = {'None': None, 'True': True, 'False': False}
if isinstance(node_or_string, str):
node_or_string = parse(node_or_string, mode='eval')
if isinstance(node_or_string, Expression):
@@ -61,9 +60,8 @@
elif isinstance(node, Dict):
return dict((_convert(k), _convert(v)) for k, v
in zip(node.keys, node.values))
- elif isinstance(node, Name):
- if node.id in _safe_names:
- return _safe_names[node.id]
+ elif isinstance(node, NameConstant):
+ return node.value
elif isinstance(node, UnaryOp) and \
isinstance(node.op, (UAdd, USub)) and \
isinstance(node.operand, (Num, UnaryOp, BinOp)):
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -928,6 +928,9 @@
def test_tuple(self):
self._sequence(ast.Tuple)
+ def test_nameconstant(self):
+ self.expr(ast.NameConstant(4), "singleton must be True, False, or None")
+
def test_stdlib_validates(self):
stdlib = os.path.dirname(ast.__file__)
tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")]
@@ -959,13 +962,13 @@
#### EVERYTHING BELOW IS GENERATED #####
exec_results = [
-('Module', [('Expr', (1, 0), ('Name', (1, 0), 'None', ('Load',)))]),
+('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [], []), [('Pass', (1, 10))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None)], None, None, [], None, None, [('Num', (1, 8), 0)], []), [('Pass', (1, 12))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], 'args', None, [], None, None, [], []), [('Pass', (1, 14))], [], None)]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], 'kwargs', None, [], []), [('Pass', (1, 17))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('Name', (1, 16), 'None', ('Load',)), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', 'a', None), ('arg', 'b', None), ('arg', 'c', None), ('arg', 'd', None), ('arg', 'e', None)], 'args', None, [], 'kwargs', None, [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])], []), [('Pass', (1, 52))], [], None)]),
('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]),
('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]),
('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
@@ -1002,14 +1005,14 @@
('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]),
]
eval_results = [
-('Expression', ('Name', (1, 0), 'None', ('Load',))),
+('Expression', ('NameConstant', (1, 0), None)),
('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])),
('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
-('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))),
+('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('NameConstant', (1, 7), None))),
('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
('Expression', ('Dict', (1, 0), [], [])),
-('Expression', ('Set', (1, 0), [('Name', (1, 1), 'None', ('Load',))])),
+('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])),
('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -33,7 +33,7 @@
>>> None = 1
Traceback (most recent call last):
-SyntaxError: assignment to keyword
+SyntaxError: can't assign to keyword
It's a syntax error to assign to the empty tuple. Why isn't it an
error to assign to the empty list? It will always raise some error at
@@ -233,7 +233,7 @@
SyntaxError: can't assign to generator expression
>>> None += 1
Traceback (most recent call last):
-SyntaxError: assignment to keyword
+SyntaxError: can't assign to keyword
>>> f() += 1
Traceback (most recent call last):
SyntaxError: can't assign to function call
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,10 @@
Core and Builtins
-----------------
+- Issue #16619: Create NameConstant AST class to represent None, True, and False
+ literals. As a result, these constants are never loaded at runtime from
+ builtins.
+
- Issue #16455: On FreeBSD and Solaris, if the locale is C, the
ASCII/surrogateescape codec is now used, instead of the locale encoding, to
decode the command line arguments. This change fixes inconsistencies with
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -1,4 +1,4 @@
--- ASDL's five builtin types are identifier, int, string, bytes, object
+-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton
module Python
{
@@ -69,8 +69,8 @@
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
| Bytes(bytes s)
+ | NameConstant(singleton value)
| Ellipsis
- -- other literals? bools?
-- the following expression can appear in assignment context
| Attribute(expr value, identifier attr, expr_context ctx)
diff --git a/Parser/asdl.py b/Parser/asdl.py
--- a/Parser/asdl.py
+++ b/Parser/asdl.py
@@ -222,7 +222,7 @@
" field ::= Id ? "
return Field(type[0], opt=True)
-builtin_types = ("identifier", "string", "bytes", "int", "object")
+builtin_types = ("identifier", "string", "bytes", "int", "object", "singleton")
# below is a collection of classes to capture the AST of an AST :-)
# not sure if any of the methods are useful yet, but I'm adding them
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -820,6 +820,7 @@
Py_INCREF((PyObject*)o);
return (PyObject*)o;
}
+#define ast2obj_singleton ast2obj_object
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
#define ast2obj_bytes ast2obj_object
@@ -831,6 +832,17 @@
/* Conversion Python -> AST */
+static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
+{
+ if (obj != Py_None && obj != Py_True && obj != Py_False) {
+ PyErr_SetString(PyExc_ValueError,
+ "AST singleton must be True, False, or None");
+ return 1;
+ }
+ *out = obj;
+ return 0;
+}
+
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
{
if (obj == Py_None)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -271,6 +271,10 @@
static char *Bytes_fields[]={
"s",
};
+static PyTypeObject *NameConstant_type;
+static char *NameConstant_fields[]={
+ "value",
+};
static PyTypeObject *Ellipsis_type;
static PyTypeObject *Attribute_type;
_Py_IDENTIFIER(attr);
@@ -673,6 +677,7 @@
Py_INCREF((PyObject*)o);
return (PyObject*)o;
}
+#define ast2obj_singleton ast2obj_object
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
#define ast2obj_bytes ast2obj_object
@@ -684,6 +689,17 @@
/* Conversion Python -> AST */
+static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
+{
+ if (obj != Py_None && obj != Py_True && obj != Py_False) {
+ PyErr_SetString(PyExc_ValueError,
+ "AST singleton must be True, False, or None");
+ return 1;
+ }
+ *out = obj;
+ return 0;
+}
+
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
{
if (obj == Py_None)
@@ -860,6 +876,9 @@
if (!Str_type) return 0;
Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1);
if (!Bytes_type) return 0;
+ NameConstant_type = make_type("NameConstant", expr_type,
+ NameConstant_fields, 1);
+ if (!NameConstant_type) return 0;
Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0);
if (!Ellipsis_type) return 0;
Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3);
@@ -1921,6 +1940,25 @@
}
expr_ty
+NameConstant(singleton value, int lineno, int col_offset, PyArena *arena)
+{
+ expr_ty p;
+ if (!value) {
+ PyErr_SetString(PyExc_ValueError,
+ "field value is required for NameConstant");
+ return NULL;
+ }
+ p = (expr_ty)PyArena_Malloc(arena, sizeof(*p));
+ if (!p)
+ return NULL;
+ p->kind = NameConstant_kind;
+ p->v.NameConstant.value = value;
+ p->lineno = lineno;
+ p->col_offset = col_offset;
+ return p;
+}
+
+expr_ty
Ellipsis(int lineno, int col_offset, PyArena *arena)
{
expr_ty p;
@@ -2028,6 +2066,7 @@
*arena)
{
expr_ty p;
+ assert(PyUnicode_CompareWithASCIIString(id, "True") && PyUnicode_CompareWithASCIIString(id, "False") && PyUnicode_CompareWithASCIIString(id, "None"));
if (!id) {
PyErr_SetString(PyExc_ValueError,
"field id is required for Name");
@@ -2948,6 +2987,15 @@
goto failed;
Py_DECREF(value);
break;
+ case NameConstant_kind:
+ result = PyType_GenericNew(NameConstant_type, NULL, NULL);
+ if (!result) goto failed;
+ value = ast2obj_singleton(o->v.NameConstant.value);
+ if (!value) goto failed;
+ if (_PyObject_SetAttrId(result, &PyId_value, value) == -1)
+ goto failed;
+ Py_DECREF(value);
+ break;
case Ellipsis_kind:
result = PyType_GenericNew(Ellipsis_type, NULL, NULL);
if (!result) goto failed;
@@ -5688,6 +5736,29 @@
if (*out == NULL) goto failed;
return 0;
}
+ isinstance = PyObject_IsInstance(obj, (PyObject*)NameConstant_type);
+ if (isinstance == -1) {
+ return 1;
+ }
+ if (isinstance) {
+ singleton value;
+
+ if (_PyObject_HasAttrId(obj, &PyId_value)) {
+ int res;
+ tmp = _PyObject_GetAttrId(obj, &PyId_value);
+ if (tmp == NULL) goto failed;
+ res = obj2ast_singleton(tmp, &value, arena);
+ if (res != 0) goto failed;
+ Py_XDECREF(tmp);
+ tmp = NULL;
+ } else {
+ PyErr_SetString(PyExc_TypeError, "required field \"value\" missing from NameConstant");
+ return 1;
+ }
+ *out = NameConstant(value, lineno, col_offset, arena);
+ if (*out == NULL) goto failed;
+ return 0;
+ }
isinstance = PyObject_IsInstance(obj, (PyObject*)Ellipsis_type);
if (isinstance == -1) {
return 1;
@@ -7008,6 +7079,8 @@
NULL;
if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return
NULL;
+ if (PyDict_SetItemString(d, "NameConstant",
+ (PyObject*)NameConstant_type) < 0) return NULL;
if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0)
return NULL;
if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) <
diff --git a/Python/ast.c b/Python/ast.c
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -282,6 +282,7 @@
return validate_exprs(exp->v.Tuple.elts, ctx, 0);
/* These last cases don't have any checking. */
case Name_kind:
+ case NameConstant_kind:
case Ellipsis_kind:
return 1;
default:
@@ -903,7 +904,7 @@
break;
case Name_kind:
if (ctx == Store) {
- if (forbidden_name(c, e->v.Name.id, n, 1))
+ if (forbidden_name(c, e->v.Name.id, n, 0))
return 0; /* forbidden_name() calls ast_error() */
}
e->v.Name.ctx = ctx;
@@ -955,6 +956,9 @@
case Bytes_kind:
expr_name = "literal";
break;
+ case NameConstant_kind:
+ expr_name = "keyword";
+ break;
case Ellipsis_kind:
expr_name = "Ellipsis";
break;
@@ -1819,11 +1823,21 @@
switch (TYPE(ch)) {
case NAME: {
- /* All names start in Load context, but may later be
- changed. */
- PyObject *name = NEW_IDENTIFIER(ch);
+ PyObject *name;
+ const char *s = STR(ch);
+ size_t len = strlen(s);
+ if (len >= 4 && len <= 5) {
+ if (!strcmp(s, "None"))
+ return NameConstant(Py_None, LINENO(n), n->n_col_offset, c->c_arena);
+ if (!strcmp(s, "True"))
+ return NameConstant(Py_True, LINENO(n), n->n_col_offset, c->c_arena);
+ if (!strcmp(s, "False"))
+ return NameConstant(Py_False, LINENO(n), n->n_col_offset, c->c_arena);
+ }
+ name = new_identifier(s, c);
if (!name)
return NULL;
+ /* All names start in Load context, but may later be changed. */
return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena);
}
case STRING: {
diff --git a/Python/compile.c b/Python/compile.c
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -3194,12 +3194,18 @@
case Name_kind:
/* optimize away names that can't be reassigned */
id = PyUnicode_AsUTF8(e->v.Name.id);
- if (strcmp(id, "True") == 0) return 1;
- if (strcmp(id, "False") == 0) return 0;
- if (strcmp(id, "None") == 0) return 0;
- if (strcmp(id, "__debug__") == 0)
- return ! c->c_optimize;
- /* fall through */
+ if (id && strcmp(id, "__debug__") == 0)
+ return !c->c_optimize;
+ return -1;
+ case NameConstant_kind: {
+ PyObject *o = e->v.NameConstant.value;
+ if (o == Py_None)
+ return 0;
+ else if (o == Py_True)
+ return 1;
+ else if (o == Py_False)
+ return 0;
+ }
default:
return -1;
}
@@ -3375,6 +3381,9 @@
case Ellipsis_kind:
ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts);
break;
+ case NameConstant_kind:
+ ADDOP_O(c, LOAD_CONST, e->v.NameConstant.value, consts);
+ break;
/* The following exprs can be assignment targets. */
case Attribute_kind:
if (e->v.Attribute.ctx != AugStore)
diff --git a/Python/peephole.c b/Python/peephole.c
--- a/Python/peephole.c
+++ b/Python/peephole.c
@@ -327,37 +327,6 @@
return blocks;
}
-/* Helper to replace LOAD_NAME None/True/False with LOAD_CONST
- Returns: 0 if no change, 1 if change, -1 if error */
-static int
-load_global(unsigned char *codestr, Py_ssize_t i, char *name, PyObject *consts)
-{
- Py_ssize_t j;
- PyObject *obj;
- if (name == NULL)
- return 0;
- if (strcmp(name, "None") == 0)
- obj = Py_None;
- else if (strcmp(name, "True") == 0)
- obj = Py_True;
- else if (strcmp(name, "False") == 0)
- obj = Py_False;
- else
- return 0;
- for (j = 0; j < PyList_GET_SIZE(consts); j++) {
- if (PyList_GET_ITEM(consts, j) == obj)
- break;
- }
- if (j == PyList_GET_SIZE(consts)) {
- if (PyList_Append(consts, obj) < 0)
- return -1;
- }
- assert(PyList_GET_ITEM(consts, j) == obj);
- codestr[i] = LOAD_CONST;
- SETARG(codestr, i, j);
- return 1;
-}
-
/* Perform basic peephole optimizations to components of a code object.
The consts object should still be in list form to allow new constants
to be appended.
@@ -392,7 +361,6 @@
Py_ssize_t const_stack_size = 0;
int in_consts = 0; /* whether we are in a LOAD_CONST sequence */
unsigned int *blocks = NULL;
- char *name;
/* Bail out if an exception is set */
if (PyErr_Occurred())
@@ -475,20 +443,6 @@
codestr[i+3] = NOP;
break;
- /* Replace LOAD_GLOBAL/LOAD_NAME None/True/False
- with LOAD_CONST None/True/False */
- case LOAD_NAME:
- case LOAD_GLOBAL:
- j = GETARG(codestr, i);
- name = _PyUnicode_AsString(PyTuple_GET_ITEM(names, j));
- h = load_global(codestr, i, name, consts);
- if (h < 0)
- goto exitError;
- else if (h == 0)
- continue;
- CONST_STACK_PUSH_OP(i);
- break;
-
/* Skip over LOAD_CONST trueconst
POP_JUMP_IF_FALSE xx. This improves
"while 1" performance. */
diff --git a/Python/symtable.c b/Python/symtable.c
--- a/Python/symtable.c
+++ b/Python/symtable.c
@@ -1437,6 +1437,7 @@
case Str_kind:
case Bytes_kind:
case Ellipsis_kind:
+ case NameConstant_kind:
/* Nothing to do here. */
break;
/* The following exprs can be assignment targets. */
diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py
--- a/Tools/parser/unparse.py
+++ b/Tools/parser/unparse.py
@@ -307,6 +307,9 @@
def _Name(self, t):
self.write(t.id)
+ def _NameConstant(self, t):
+ self.write(repr(t.value))
+
def _Num(self, t):
# Substitute overflowing decimal literal for AST infinities.
self.write(repr(t.n).replace("inf", INFSTR))
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list