[Python-checkins] cpython (merge 3.4 -> default): Issue #23374: Fixed pydoc failure with non-ASCII files when stdout encoding

[serhiy.storchaka]

https://hg.python.org/cpython/rev/affe167a45f3
changeset:   94707:affe167a45f3
parent:      94705:56c6a4bce996
parent:      94706:e7b6b1f57268
user:        Serhiy Storchaka <storchaka at gmail.com>
date:        Fri Feb 20 23:47:09 2015 +0200
summary:
  Issue #23374: Fixed pydoc failure with non-ASCII files when stdout encoding
differs from file system encoding (e.g. on Mac OS).

files:
  Lib/pydoc.py           |  22 +++++++++++++---------
  Lib/test/test_pydoc.py |   9 +++++++++
  Misc/NEWS              |   3 +++
  3 files changed, 25 insertions(+), 9 deletions(-)


diff --git a/Lib/pydoc.py b/Lib/pydoc.py
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -1405,9 +1405,6 @@
 def pager(text):
     """The first time this is called, determine what kind of pager to use."""
     global pager
-    # Escape non-encodable characters to avoid encoding errors later
-    encoding = sys.getfilesystemencoding()
-    text = text.encode(encoding, 'backslashreplace').decode(encoding)
     pager = getpager()
     pager(text)
 
@@ -1450,10 +1447,12 @@
 
 def pipepager(text, cmd):
     """Page through text by feeding it to another program."""
-    pipe = os.popen(cmd, 'w')
+    import subprocess
+    proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE)
     try:
-        pipe.write(text)
-        pipe.close()
+        with proc:
+            with io.TextIOWrapper(proc.stdin, errors='backslashreplace') as pipe:
+                pipe.write(text)
     except OSError:
         pass # Ignore broken pipes caused by quitting the pager program.
 
@@ -1461,16 +1460,21 @@
     """Page through text by invoking a program on a temporary file."""
     import tempfile
     filename = tempfile.mktemp()
-    with open(filename, 'w') as file:
+    with open(filename, 'w', errors='backslashreplace') as file:
         file.write(text)
     try:
         os.system(cmd + ' "' + filename + '"')
     finally:
         os.unlink(filename)
 
+def _escape_stdout(text):
+    # Escape non-encodable characters to avoid encoding errors later
+    encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
+    return text.encode(encoding, 'backslashreplace').decode(encoding)
+
 def ttypager(text):
     """Page through text on a text terminal."""
-    lines = plain(text).split('\n')
+    lines = plain(_escape_stdout(text)).split('\n')
     try:
         import tty
         fd = sys.stdin.fileno()
@@ -1514,7 +1518,7 @@
 
 def plainpager(text):
     """Simply print unformatted text.  This is the ultimate fallback."""
-    sys.stdout.write(plain(text))
+    sys.stdout.write(plain(_escape_stdout(text)))
 
 def describe(thing):
     """Produce a short description of the given thing."""
diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py
--- a/Lib/test/test_pydoc.py
+++ b/Lib/test/test_pydoc.py
@@ -34,6 +34,10 @@
 except ImportError:
     threading = None
 
+class nonascii:
+    'Це не латиниця'
+    pass
+
 if test.support.HAVE_DOCSTRINGS:
     expected_data_docstrings = (
         'dictionary for instance variables (if defined)',
@@ -460,6 +464,11 @@
         self.assertEqual(expected, result,
             "documentation for missing module found")
 
+    def test_not_ascii(self):
+        result = run_pydoc('test.test_pydoc.nonascii', PYTHONIOENCODING='ascii')
+        encoded = nonascii.__doc__.encode('ascii', 'backslashreplace')
+        self.assertIn(encoded, result)
+
     def test_input_strip(self):
         missing_module = " test.i_am_not_here "
         result = str(run_pydoc(missing_module), 'ascii')
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -13,6 +13,9 @@
 Library
 -------
 
+- Issue #23374: Fixed pydoc failure with non-ASCII files when stdout encoding
+  differs from file system encoding (e.g. on Mac OS).
+
 - Issue #23481: Remove RC4 from the SSL module's default cipher list.
 
 - Issue #21548: Fix pydoc.synopsis() and pydoc.apropos() on modules with empty

-- 
Repository URL: https://hg.python.org/cpython