[Pytest-commit] commit/pytest: 4 new changesets

commits-noreply at bitbucket.org commits-noreply at bitbucket.org
Thu Aug 7 10:43:46 CEST 2014


4 new commits in pytest:

https://bitbucket.org/hpk42/pytest/commits/1fe6e5cab490/
Changeset:   1fe6e5cab490
Branch:      assertionrewrite-currupted-pyc
User:        nicoddemus
Date:        2014-08-02 23:01:28
Summary:     Fixed assertionrewrite._read_pyc to handle corrupted pyc files properly

This seems to be the cause for issues #437 and #301.
Affected #:  2 files

diff -r 9e3ed0aaa45a92b50de88a72224f69ea22b805f0 -r 1fe6e5cab490a6850b7d31c7547e78e7b3d2eef7 _pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -308,7 +308,10 @@
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("<l", data[4:])[0] != mtime):
             return None
-        co = marshal.load(fp)
+        try:
+            co = marshal.load(fp)
+        except (EOFError, ValueError, TypeError):  # see docs
+            return None
         if not isinstance(co, types.CodeType):
             # That's interesting....
             return None

diff -r 9e3ed0aaa45a92b50de88a72224f69ea22b805f0 -r 1fe6e5cab490a6850b7d31c7547e78e7b3d2eef7 testing/test_assertrewrite.py
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -539,3 +539,25 @@
         result.stdout.fnmatch_lines([
             '* 1 passed*',
         ])
+
+    def test_read_pyc(self, tmpdir):
+        """
+        Ensure that the `_read_pyc` can properly deal with corrupted pyc files.
+        In those circumstances it should just give up instead of generating
+        an exception that is propagated to the caller.
+        """
+        import py_compile
+        from _pytest.assertion.rewrite import _read_pyc
+
+        source = tmpdir.join('source.py')
+        pyc = source + 'c'
+
+        source.write('def test(): pass')
+        py_compile.compile(str(source), str(pyc))
+
+        contents = pyc.read(mode='rb')
+        strip_bytes = 20  # header is around 8 bytes, strip a little more
+        assert len(contents) > strip_bytes
+        pyc.write(contents[:strip_bytes], mode='wb')
+
+        assert _read_pyc(source, str(pyc)) is None  # no error


https://bitbucket.org/hpk42/pytest/commits/a07708939a75/
Changeset:   a07708939a75
Branch:      assertionrewrite-currupted-pyc
User:        nicoddemus
Date:        2014-08-05 01:38:50
Summary:     updated CHANGELOG and trace error message as requested in review

fixes issue #437
Affected #:  2 files

diff -r 1fe6e5cab490a6850b7d31c7547e78e7b3d2eef7 -r a07708939a75cd1c50d7724af1e5933a449a2b32 CHANGELOG
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
 NEXT
 -----------------------------------
 
+- fix issue437 where assertion rewriting could cause pytest-xdist slaves
+  to collect different tests.
+
 - fix issue547 capsys/capfd also work when output capturing ("-s") is disabled.
 
 - address issue170: allow pytest.mark.xfail(...) to specify expected exceptions via

diff -r 1fe6e5cab490a6850b7d31c7547e78e7b3d2eef7 -r a07708939a75cd1c50d7724af1e5933a449a2b32 _pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -131,7 +131,7 @@
         pyc = os.path.join(cache_dir, cache_name)
         # Notice that even if we're in a read-only directory, I'm going
         # to check for a cached pyc. This may not be optimal...
-        co = _read_pyc(fn_pypath, pyc)
+        co = _read_pyc(fn_pypath, pyc, state.trace)
         if co is None:
             state.trace("rewriting %r" % (fn,))
             co = _rewrite_test(state, fn_pypath)
@@ -289,11 +289,13 @@
         if _write_pyc(state, co, fn, proc_pyc):
             os.rename(proc_pyc, pyc)
 
-def _read_pyc(source, pyc):
+def _read_pyc(source, pyc, trace=None):
     """Possibly read a pytest pyc containing rewritten code.
 
     Return rewritten code if successful or None if not.
     """
+    if trace is None:
+        trace = lambda x: None
     try:
         fp = open(pyc, "rb")
     except IOError:
@@ -302,18 +304,21 @@
         try:
             mtime = int(source.mtime())
             data = fp.read(8)
-        except EnvironmentError:
+        except EnvironmentError as e:
+            trace('_read_pyc(%s): EnvironmentError %s' % (source, e))
             return None
         # Check for invalid or out of date pyc file.
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("<l", data[4:])[0] != mtime):
+            trace('_read_pyc(%s): invalid or out of date pyc' % source)
             return None
         try:
             co = marshal.load(fp)
-        except (EOFError, ValueError, TypeError):  # see docs
+        except Exception as e:
+            trace('_read_pyc(%s): marshal.load error %s' % (source, e))
             return None
         if not isinstance(co, types.CodeType):
-            # That's interesting....
+            trace('_read_pyc(%s): not a code object' % source)
             return None
         return co
     finally:


https://bitbucket.org/hpk42/pytest/commits/082460583d95/
Changeset:   082460583d95
User:        hpk42
Date:        2014-08-07 10:42:23
Summary:     merge PR192, streamline a bit.
Affected #:  3 files

diff -r eb6f0a6eb8a98771c8b81d2028fc39cec783c52d -r 082460583d9567f5f56ff79e9cefdc2db7824697 CHANGELOG
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -3,6 +3,10 @@
 
 - No longer show line numbers in the --verbose output, the output is now
   purely the nodeid.  The line number is still shown in failure reports.
+  Thanks Floris Bruynooghe.
+
+- fix issue437 where assertion rewriting could cause pytest-xdist slaves
+  to collect different tests. Thanks Bruno Oliveira.
 
 - fix issue547 capsys/capfd also work when output capturing ("-s") is disabled.
 

diff -r eb6f0a6eb8a98771c8b81d2028fc39cec783c52d -r 082460583d9567f5f56ff79e9cefdc2db7824697 _pytest/assertion/rewrite.py
--- a/_pytest/assertion/rewrite.py
+++ b/_pytest/assertion/rewrite.py
@@ -131,7 +131,7 @@
         pyc = os.path.join(cache_dir, cache_name)
         # Notice that even if we're in a read-only directory, I'm going
         # to check for a cached pyc. This may not be optimal...
-        co = _read_pyc(fn_pypath, pyc)
+        co = _read_pyc(fn_pypath, pyc, state.trace)
         if co is None:
             state.trace("rewriting %r" % (fn,))
             co = _rewrite_test(state, fn_pypath)
@@ -289,7 +289,7 @@
         if _write_pyc(state, co, fn, proc_pyc):
             os.rename(proc_pyc, pyc)
 
-def _read_pyc(source, pyc):
+def _read_pyc(source, pyc, trace=lambda x: None):
     """Possibly read a pytest pyc containing rewritten code.
 
     Return rewritten code if successful or None if not.
@@ -298,23 +298,27 @@
         fp = open(pyc, "rb")
     except IOError:
         return None
-    try:
+    with fp:
         try:
             mtime = int(source.mtime())
             data = fp.read(8)
-        except EnvironmentError:
+        except EnvironmentError as e:
+            trace('_read_pyc(%s): EnvironmentError %s' % (source, e))
             return None
         # Check for invalid or out of date pyc file.
         if (len(data) != 8 or data[:4] != imp.get_magic() or
                 struct.unpack("<l", data[4:])[0] != mtime):
+            trace('_read_pyc(%s): invalid or out of date pyc' % source)
             return None
-        co = marshal.load(fp)
+        try:
+            co = marshal.load(fp)
+        except Exception as e:
+            trace('_read_pyc(%s): marshal.load error %s' % (source, e))
+            return None
         if not isinstance(co, types.CodeType):
-            # That's interesting....
+            trace('_read_pyc(%s): not a code object' % source)
             return None
         return co
-    finally:
-        fp.close()
 
 
 def rewrite_asserts(mod):

diff -r eb6f0a6eb8a98771c8b81d2028fc39cec783c52d -r 082460583d9567f5f56ff79e9cefdc2db7824697 testing/test_assertrewrite.py
--- a/testing/test_assertrewrite.py
+++ b/testing/test_assertrewrite.py
@@ -539,3 +539,25 @@
         result.stdout.fnmatch_lines([
             '* 1 passed*',
         ])
+
+    def test_read_pyc(self, tmpdir):
+        """
+        Ensure that the `_read_pyc` can properly deal with corrupted pyc files.
+        In those circumstances it should just give up instead of generating
+        an exception that is propagated to the caller.
+        """
+        import py_compile
+        from _pytest.assertion.rewrite import _read_pyc
+
+        source = tmpdir.join('source.py')
+        pyc = source + 'c'
+
+        source.write('def test(): pass')
+        py_compile.compile(str(source), str(pyc))
+
+        contents = pyc.read(mode='rb')
+        strip_bytes = 20  # header is around 8 bytes, strip a little more
+        assert len(contents) > strip_bytes
+        pyc.write(contents[:strip_bytes], mode='wb')
+
+        assert _read_pyc(source, str(pyc)) is None  # no error


https://bitbucket.org/hpk42/pytest/commits/cc7f185b717e/
Changeset:   cc7f185b717e
Branch:      assertionrewrite-currupted-pyc
User:        hpk42
Date:        2014-08-07 10:42:34
Summary:     close branch
Affected #:  0 files

Repository URL: https://bitbucket.org/hpk42/pytest/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.


More information about the pytest-commit mailing list