[pypy-commit] pypy default: issue 1762: accept null bytes in the .py file named in the command-line,

[arigo]

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r71564:ec637e30bcd6
Date: 2014-05-18 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ec637e30bcd6/

Log:	issue 1762: accept null bytes in the .py file named in the command-
	line, in addition to files that are imported.

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -590,6 +590,11 @@
             # handle the case where no command/filename/module is specified
             # on the command-line.
 
+            try:
+                from _ast import PyCF_ACCEPT_NULL_BYTES
+            except ImportError:
+                PyCF_ACCEPT_NULL_BYTES = 0
+
             # update sys.path *after* loading site.py, in case there is a
             # "site.py" file in the script's directory. Only run this if we're
             # executing the interactive prompt, if we're running a script we
@@ -613,7 +618,8 @@
                         def run_it():
                             co_python_startup = compile(startup,
                                                         python_startup,
-                                                        'exec')
+                                                        'exec',
+                                                        PyCF_ACCEPT_NULL_BYTES)
                             exec co_python_startup in mainmodule.__dict__
                         mainmodule.__file__ = python_startup
                         run_toplevel(run_it)
@@ -626,7 +632,8 @@
             else:
                 # If not interactive, just read and execute stdin normally.
                 def run_it():
-                    co_stdin = compile(sys.stdin.read(), '<stdin>', 'exec')
+                    co_stdin = compile(sys.stdin.read(), '<stdin>', 'exec',
+                                       PyCF_ACCEPT_NULL_BYTES)
                     exec co_stdin in mainmodule.__dict__
                 mainmodule.__file__ = '<stdin>'
                 success = run_toplevel(run_it)
diff --git a/pypy/interpreter/astcompiler/consts.py b/pypy/interpreter/astcompiler/consts.py
--- a/pypy/interpreter/astcompiler/consts.py
+++ b/pypy/interpreter/astcompiler/consts.py
@@ -22,3 +22,4 @@
 PyCF_SOURCE_IS_UTF8 = 0x0100
 PyCF_DONT_IMPLY_DEDENT = 0x0200
 PyCF_ONLY_AST = 0x0400
+PyCF_ACCEPT_NULL_BYTES = 0x10000000   # PyPy only, for compile()
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -24,7 +24,8 @@
 """
     ec = space.getexecutioncontext()
     if flags & ~(ec.compiler.compiler_flags | consts.PyCF_ONLY_AST |
-                 consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8):
+                 consts.PyCF_DONT_IMPLY_DEDENT | consts.PyCF_SOURCE_IS_UTF8 |
+                 consts.PyCF_ACCEPT_NULL_BYTES):
         raise OperationError(space.w_ValueError,
                              space.wrap("compile() unrecognized flags"))
 
@@ -53,9 +54,10 @@
     else:
         source = space.readbuf_w(w_source).as_str()
 
-    if '\x00' in source:
-        raise OperationError(space.w_TypeError, space.wrap(
-            "compile() expected string without null bytes"))
+    if not (flags & consts.PyCF_ACCEPT_NULL_BYTES):
+        if '\x00' in source:
+            raise OperationError(space.w_TypeError, space.wrap(
+                "compile() expected string without null bytes"))
 
     if flags & consts.PyCF_ONLY_AST:
         code = ec.compiler.compile_to_ast(source, filename, mode, flags)
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -610,6 +610,16 @@
         firstlineno = co.co_firstlineno
         assert firstlineno == 2
 
+    def test_compile_null_bytes(self):
+        import _ast
+        raises(TypeError, compile, '\x00', 'mymod', 'exec', 0)
+        raises(SyntaxError, compile, '\x00', 'mymod', 'exec',
+               _ast.PyCF_ACCEPT_NULL_BYTES)
+        src = "#abc\x00def\n"
+        raises(TypeError, compile, src, 'mymod', 'exec')
+        raises(TypeError, compile, src, 'mymod', 'exec', 0)
+        compile(src, 'mymod', 'exec', _ast.PyCF_ACCEPT_NULL_BYTES)  # works
+
     def test_print_function(self):
         import __builtin__
         import sys
diff --git a/pypy/module/_ast/__init__.py b/pypy/module/_ast/__init__.py
--- a/pypy/module/_ast/__init__.py
+++ b/pypy/module/_ast/__init__.py
@@ -6,6 +6,8 @@
 
     interpleveldefs = {
         "PyCF_ONLY_AST" : "space.wrap(%s)" % consts.PyCF_ONLY_AST,
+        "PyCF_ACCEPT_NULL_BYTES":
+                          "space.wrap(%s)" % consts.PyCF_ACCEPT_NULL_BYTES,
         "__version__"   : "space.wrap('82160')",  # from CPython's svn.
         }
     appleveldefs = {}