[Python-checkins] r70113 - in python/branches/py3k: Include/parsetok.h Include/pythonrun.h Lib/test/test_coding.py Lib/test/test_pep263.py Misc/NEWS Parser/parsetok.c Parser/tokenizer.c Parser/tokenizer.h Python/bltinmodule.c Python/pythonrun.c
benjamin.peterson
python-checkins at python.org
Tue Mar 3 00:31:26 CET 2009
Author: benjamin.peterson
Date: Tue Mar 3 00:31:26 2009
New Revision: 70113
Log:
ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
Modified:
python/branches/py3k/Include/parsetok.h
python/branches/py3k/Include/pythonrun.h
python/branches/py3k/Lib/test/test_coding.py
python/branches/py3k/Lib/test/test_pep263.py
python/branches/py3k/Misc/NEWS
python/branches/py3k/Parser/parsetok.c
python/branches/py3k/Parser/tokenizer.c
python/branches/py3k/Parser/tokenizer.h
python/branches/py3k/Python/bltinmodule.c
python/branches/py3k/Python/pythonrun.c
Modified: python/branches/py3k/Include/parsetok.h
==============================================================================
--- python/branches/py3k/Include/parsetok.h (original)
+++ python/branches/py3k/Include/parsetok.h Tue Mar 3 00:31:26 2009
@@ -29,6 +29,8 @@
#define PyPARSE_UNICODE_LITERALS 0x0008
#endif
+#define PyPARSE_IGNORE_COOKIE 0x0010
+
PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
perrdetail *);
PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
Modified: python/branches/py3k/Include/pythonrun.h
==============================================================================
--- python/branches/py3k/Include/pythonrun.h (original)
+++ python/branches/py3k/Include/pythonrun.h Tue Mar 3 00:31:26 2009
@@ -12,6 +12,7 @@
#define PyCF_SOURCE_IS_UTF8 0x0100
#define PyCF_DONT_IMPLY_DEDENT 0x0200
#define PyCF_ONLY_AST 0x0400
+#define PyCF_IGNORE_COOKIE 0x0800
typedef struct {
int cf_flags; /* bitmask of CO_xxx flags relevant to future */
Modified: python/branches/py3k/Lib/test/test_coding.py
==============================================================================
--- python/branches/py3k/Lib/test/test_coding.py (original)
+++ python/branches/py3k/Lib/test/test_coding.py Tue Mar 3 00:31:26 2009
@@ -17,10 +17,10 @@
path = os.path.dirname(__file__)
filename = os.path.join(path, module_name + '.py')
- fp = open(filename, encoding='utf-8')
- text = fp.read()
+ fp = open(filename, "rb")
+ bytes = fp.read()
fp.close()
- self.assertRaises(SyntaxError, compile, text, filename, 'exec')
+ self.assertRaises(SyntaxError, compile, bytes, filename, 'exec')
def test_exec_valid_coding(self):
d = {}
Modified: python/branches/py3k/Lib/test/test_pep263.py
==============================================================================
--- python/branches/py3k/Lib/test/test_pep263.py (original)
+++ python/branches/py3k/Lib/test/test_pep263.py Tue Mar 3 00:31:26 2009
@@ -30,6 +30,12 @@
else:
self.fail()
+ def test_issue4626(self):
+ c = compile("# coding=latin-1\n\u00c6 = '\u00c6'", "dummy", "exec")
+ d = {}
+ exec(c, d)
+ self.assertEquals(d['\xc6'], '\xc6')
+
def test_main():
support.run_unittest(PEP263Test)
Modified: python/branches/py3k/Misc/NEWS
==============================================================================
--- python/branches/py3k/Misc/NEWS (original)
+++ python/branches/py3k/Misc/NEWS Tue Mar 3 00:31:26 2009
@@ -19,6 +19,9 @@
- Issue #5249: time.strftime returned malformed string when format string
contained non ascii character on windows.
+- Issue #4626: compile(), exec(), and eval() ignore the coding cookie if the
+ source has already been decoded into str.
+
- Issue #5186: Reduce hash collisions for objects with no __hash__ method by
rotating the object pointer by 4 bits to the right.
Modified: python/branches/py3k/Parser/parsetok.c
==============================================================================
--- python/branches/py3k/Parser/parsetok.c (original)
+++ python/branches/py3k/Parser/parsetok.c Tue Mar 3 00:31:26 2009
@@ -49,7 +49,11 @@
initerr(err_ret, filename);
- if ((tok = PyTokenizer_FromString(s)) == NULL) {
+ if (*flags & PyPARSE_IGNORE_COOKIE)
+ tok = PyTokenizer_FromUTF8(s);
+ else
+ tok = PyTokenizer_FromString(s);
+ if (tok == NULL) {
err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
return NULL;
}
Modified: python/branches/py3k/Parser/tokenizer.c
==============================================================================
--- python/branches/py3k/Parser/tokenizer.c (original)
+++ python/branches/py3k/Parser/tokenizer.c Tue Mar 3 00:31:26 2009
@@ -715,6 +715,28 @@
return tok;
}
+struct tok_state *
+PyTokenizer_FromUTF8(const char *str)
+{
+ struct tok_state *tok = tok_new();
+ if (tok == NULL)
+ return NULL;
+ tok->decoding_state = STATE_RAW;
+ tok->read_coding_spec = 1;
+ tok->enc = NULL;
+ tok->str = str;
+ tok->encoding = (char *)PyMem_MALLOC(6);
+ if (!tok->encoding) {
+ PyTokenizer_Free(tok);
+ return NULL;
+ }
+ strcpy(tok->encoding, "utf-8");
+
+ /* XXX: constify members. */
+ tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
+ return tok;
+}
+
/* Set up tokenizer for file */
Modified: python/branches/py3k/Parser/tokenizer.h
==============================================================================
--- python/branches/py3k/Parser/tokenizer.h (original)
+++ python/branches/py3k/Parser/tokenizer.h Tue Mar 3 00:31:26 2009
@@ -61,6 +61,7 @@
};
extern struct tok_state *PyTokenizer_FromString(const char *);
+extern struct tok_state *PyTokenizer_FromUTF8(const char *);
extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
char *, char *);
extern void PyTokenizer_Free(struct tok_state *);
Modified: python/branches/py3k/Python/bltinmodule.c
==============================================================================
--- python/branches/py3k/Python/bltinmodule.c (original)
+++ python/branches/py3k/Python/bltinmodule.c Tue Mar 3 00:31:26 2009
@@ -494,12 +494,13 @@
static char *
-source_as_string(PyObject *cmd, char *funcname, char *what)
+source_as_string(PyObject *cmd, char *funcname, char *what, PyCompilerFlags *cf)
{
char *str;
Py_ssize_t size;
if (PyUnicode_Check(cmd)) {
+ cf->cf_flags |= PyCF_IGNORE_COOKIE;
cmd = _PyUnicode_AsDefaultEncodedString(cmd, NULL);
if (cmd == NULL)
return NULL;
@@ -591,7 +592,7 @@
return result;
}
- str = source_as_string(cmd, "compile", "string, bytes, AST or code");
+ str = source_as_string(cmd, "compile", "string, bytes, AST or code", &cf);
if (str == NULL)
return NULL;
@@ -703,14 +704,14 @@
return PyEval_EvalCode((PyCodeObject *) cmd, globals, locals);
}
- str = source_as_string(cmd, "eval", "string, bytes or code");
+ cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+ str = source_as_string(cmd, "eval", "string, bytes or code", &cf);
if (str == NULL)
return NULL;
while (*str == ' ' || *str == '\t')
str++;
- cf.cf_flags = PyCF_SOURCE_IS_UTF8;
(void)PyEval_MergeCompilerFlags(&cf);
result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);
Py_XDECREF(tmp);
@@ -779,12 +780,13 @@
v = PyEval_EvalCode((PyCodeObject *) prog, globals, locals);
}
else {
- char *str = source_as_string(prog, "exec",
- "string, bytes or code");
+ char *str;
PyCompilerFlags cf;
+ cf.cf_flags = PyCF_SOURCE_IS_UTF8;
+ str = source_as_string(prog, "exec",
+ "string, bytes or code", &cf);
if (str == NULL)
return NULL;
- cf.cf_flags = PyCF_SOURCE_IS_UTF8;
if (PyEval_MergeCompilerFlags(&cf))
v = PyRun_StringFlags(str, Py_file_input, globals,
locals, &cf);
Modified: python/branches/py3k/Python/pythonrun.c
==============================================================================
--- python/branches/py3k/Python/pythonrun.c (original)
+++ python/branches/py3k/Python/pythonrun.c Tue Mar 3 00:31:26 2009
@@ -1002,9 +1002,17 @@
}
/* compute parser flags based on compiler flags */
-#define PARSER_FLAGS(flags) \
- ((flags) ? ((((flags)->cf_flags & PyCF_DONT_IMPLY_DEDENT) ? \
- PyPARSE_DONT_IMPLY_DEDENT : 0)) : 0)
+static int PARSER_FLAGS(PyCompilerFlags *flags)
+{
+ int parser_flags = 0;
+ if (!flags)
+ return 0;
+ if (flags->cf_flags & PyCF_DONT_IMPLY_DEDENT)
+ parser_flags |= PyPARSE_DONT_IMPLY_DEDENT;
+ if (flags->cf_flags & PyCF_IGNORE_COOKIE)
+ parser_flags |= PyPARSE_IGNORE_COOKIE;
+ return parser_flags;
+}
#if 0
/* Keep an example of flags with future keyword support. */
More information about the Python-checkins
mailing list