I did a merge head with Victor's change in 2.7 before pushing my change. Can someone confirm if I did it right? If anything was wrong, how to correct it? Thank you, Senthil On Thu, Sep 3, 2015 at 2:51 AM, senthil.kumaran <python-checkins@python.org> wrote:

https://hg.python.org/cpython/rev/d687912d499f changeset: 97616:d687912d499f branch: 2.7 parent: 97615:cb781d3b1e6b parent: 97611:d739bc20d7b2 user: Senthil Kumaran <senthil@uthcode.com> date: Thu Sep 03 02:50:51 2015 -0700 summary: merge heads.

files: Lib/test/test_gdb.py | 168 +++++++++++++++++++++-- Lib/test/test_py3kwarn.py | 16 ++ Tools/gdb/libpython.py | 183 ++++++++++++++++++++++++- 3 files changed, 338 insertions(+), 29 deletions(-)

diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -10,21 +10,42 @@ import unittest import sysconfig

+from test import test_support from test.test_support import run_unittest, findfile

+# Is this Python configured to support threads? try: - gdb_version, _ = subprocess.Popen(["gdb", "-nx", "--version"], - stdout=subprocess.PIPE).communicate() -except OSError: - # This is what "no gdb" looks like. There may, however, be other - # errors that manifest this way too. - raise unittest.SkipTest("Couldn't find gdb on the path") -gdb_version_number = re.search("^GNU gdb [^\d]*(\d+)\.(\d)", gdb_version) -gdb_major_version = int(gdb_version_number.group(1)) -gdb_minor_version = int(gdb_version_number.group(2)) + import thread +except ImportError: + thread = None + +def get_gdb_version(): + try: + proc = subprocess.Popen(["gdb", "-nx", "--version"], + stdout=subprocess.PIPE, + universal_newlines=True) + version = proc.communicate()[0] + except OSError: + # This is what "no gdb" looks like. There may, however, be other + # errors that manifest this way too. + raise unittest.SkipTest("Couldn't find gdb on the path") + + # Regex to parse: + # 'GNU gdb (GDB; SUSE Linux Enterprise 12) 7.7\n' -> 7.7 + # 'GNU gdb (GDB) Fedora 7.9.1-17.fc22\n' -> 7.9 + # 'GNU gdb 6.1.1 [FreeBSD]\n' + match = re.search("^GNU gdb.*? (\d+)\.(\d)", version) + if match is None: + raise Exception("unable to parse GDB version: %r" % version) + return (version, int(match.group(1)), int(match.group(2))) + +gdb_version, gdb_major_version, gdb_minor_version = get_gdb_version() if gdb_major_version < 7: - raise unittest.SkipTest("gdb versions before 7.0 didn't support python embedding" - " Saw:\n" + gdb_version) + raise unittest.SkipTest("gdb versions before 7.0 didn't support python " + "embedding. Saw %s.%s:\n%s" + % (gdb_major_version, gdb_minor_version, + gdb_version)) + if sys.platform.startswith("sunos"): raise unittest.SkipTest("test doesn't work very well on Solaris")

@@ -713,20 +734,133 @@ class PyBtTests(DebuggerTests): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") - def test_basic_command(self): + def test_bt(self): 'Verify that the "py-bt" command works' bt = self.get_stack_trace(script=self.get_sample_script(), cmds_after_breakpoint=['py-bt']) self.assertMultilineMatches(bt, r'''^.* -#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) +Traceback \(most recent call first\): + File ".*gdb_sample.py", line 10, in baz + print\(42\) + File ".*gdb_sample.py", line 7, in bar baz\(a, b, c\) -#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\) + File ".*gdb_sample.py", line 4, in foo bar\(a, b, c\) -#[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 12, in <module> \(\) + File ".*gdb_sample.py", line 12, in <module> foo\(1, 2, 3\) ''')

+ @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + def test_bt_full(self): + 'Verify that the "py-bt-full" command works' + bt = self.get_stack_trace(script=self.get_sample_script(), + cmds_after_breakpoint=['py-bt-full']) + self.assertMultilineMatches(bt, + r'''^.* +#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 7, in bar \(a=1, b=2, c=3\) + baz\(a, b, c\) +#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\) + bar\(a, b, c\) +#[0-9]+ Frame 0x-?[0-9a-f]+, for file .*gdb_sample.py, line 12, in <module> \(\) + foo\(1, 2, 3\) +''') + + @unittest.skipUnless(thread, + "Python was compiled without thread support") + def test_threads(self): + 'Verify that "py-bt" indicates threads that are waiting for the GIL' + cmd = ''' +from threading import Thread + +class TestThread(Thread): + # These threads would run forever, but we'll interrupt things with the + # debugger + def run(self): + i = 0 + while 1: + i += 1 + +t = {} +for i in range(4): + t[i] = TestThread() + t[i].start() + +# Trigger a breakpoint on the main thread +print 42 + +''' + # Verify with "py-bt": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['thread apply all py-bt']) + self.assertIn('Waiting for the GIL', gdb_output) + + # Verify with "py-bt-full": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['thread apply all py-bt-full']) + self.assertIn('Waiting for the GIL', gdb_output) + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + # Some older versions of gdb will fail with + # "Cannot find new threads: generic error" + # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround + @unittest.skipUnless(thread, + "Python was compiled without thread support") + def test_gc(self): + 'Verify that "py-bt" indicates if a thread is garbage-collecting' + cmd = ('from gc import collect\n' + 'print 42\n' + 'def foo():\n' + ' collect()\n' + 'def bar():\n' + ' foo()\n' + 'bar()\n') + # Verify with "py-bt": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt'], + ) + self.assertIn('Garbage-collecting', gdb_output) + + # Verify with "py-bt-full": + gdb_output = self.get_stack_trace(cmd, + cmds_after_breakpoint=['break update_refs', 'continue', 'py-bt-full'], + ) + self.assertIn('Garbage-collecting', gdb_output) + + @unittest.skipIf(python_is_optimized(), + "Python was compiled with optimizations") + # Some older versions of gdb will fail with + # "Cannot find new threads: generic error" + # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround + @unittest.skipUnless(thread, + "Python was compiled without thread support") + def test_pycfunction(self): + 'Verify that "py-bt" displays invocations of PyCFunction instances' + # Tested function must not be defined with METH_NOARGS or METH_O, + # otherwise call_function() doesn't call PyCFunction_Call() + cmd = ('from time import gmtime\n' + 'def foo():\n' + ' gmtime(1)\n' + 'def bar():\n' + ' foo()\n' + 'bar()\n') + # Verify with "py-bt": + gdb_output = self.get_stack_trace(cmd, + breakpoint='time_gmtime', + cmds_after_breakpoint=['bt', 'py-bt'], + ) + self.assertIn('<built-in function gmtime', gdb_output) + + # Verify with "py-bt-full": + gdb_output = self.get_stack_trace(cmd, + breakpoint='time_gmtime', + cmds_after_breakpoint=['py-bt-full'], + ) + self.assertIn('#0 <built-in function gmtime', gdb_output) + + class PyPrintTests(DebuggerTests): @unittest.skipIf(python_is_optimized(), "Python was compiled with optimizations") @@ -781,6 +915,10 @@ r".*\na = 1\nb = 2\nc = 3\n.*")

def test_main(): + if test_support.verbose: + print("GDB version %s.%s:" % (gdb_major_version, gdb_minor_version)) + for line in gdb_version.splitlines(): + print(" " * 4 + line) run_unittest(PrettyPrintTests, PyListTests, StackNavigationTests, diff --git a/Lib/test/test_py3kwarn.py b/Lib/test/test_py3kwarn.py --- a/Lib/test/test_py3kwarn.py +++ b/Lib/test/test_py3kwarn.py @@ -2,6 +2,7 @@ import sys from test.test_support import check_py3k_warnings, CleanImport, run_unittest import warnings +from test import test_support

if not sys.py3kwarning: raise unittest.SkipTest('%s must be run with the -3 flag' % __name__) @@ -356,6 +357,21 @@ def check_removal(self, module_name, optional=False): """Make sure the specified module, when imported, raises a DeprecationWarning and specifies itself in the message.""" + if module_name in sys.modules: + mod = sys.modules[module_name] + filename = getattr(mod, '__file__', '') + mod = None + # the module is not implemented in C? + if not filename.endswith(('.py', '.pyc', '.pyo')): + # Issue #23375: If the module was already loaded, reimporting + # the module will not emit again the warning. The warning is + # emited when the module is loaded, but C modules cannot + # unloaded. + if test_support.verbose: + print("Cannot test the Python 3 DeprecationWarning of the " + "%s module, the C module is already loaded" + % module_name) + return with CleanImport(module_name), warnings.catch_warnings(): warnings.filterwarnings("error", ".+ (module|package) .+ removed", DeprecationWarning, __name__) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -45,6 +45,7 @@

from __future__ import print_function, with_statement import gdb +import locale import os import sys

@@ -76,6 +77,8 @@

MAX_OUTPUT_LEN=1024

+ENCODING = locale.getpreferredencoding() + class NullPyObjectPtr(RuntimeError): pass

@@ -92,6 +95,18 @@ # threshold in case the data was corrupted return xrange(safety_limit(int(val)))

+if sys.version_info[0] >= 3: + def write_unicode(file, text): + file.write(text) +else: + def write_unicode(file, text): + # Write a byte or unicode string to file. Unicode strings are encoded to + # ENCODING encoding with 'backslashreplace' error handler to avoid + # UnicodeEncodeError. + if isinstance(text, unicode): + text = text.encode(ENCODING, 'backslashreplace') + file.write(text) +

class StringTruncated(RuntimeError): pass @@ -903,7 +918,12 @@ newline character''' if self.is_optimized_out(): return '(frame information optimized out)' - with open(self.filename(), 'r') as f: + filename = self.filename() + try: + f = open(filename, 'r') + except IOError: + return None + with f: all_lines = f.readlines() # Convert from 1-based current_line_num to 0-based list offset: return all_lines[self.current_line_num()-1] @@ -914,9 +934,9 @@ return out.write('Frame 0x%x, for file %s, line %i, in %s (' % (self.as_address(), - self.co_filename, + self.co_filename.proxyval(visited), self.current_line_num(), - self.co_name)) + self.co_name.proxyval(visited))) first = True for pyop_name, pyop_value in self.iter_locals(): if not first: @@ -929,6 +949,16 @@

out.write(')')

+ def print_traceback(self): + if self.is_optimized_out(): + sys.stdout.write(' (frame information optimized out)\n') + return + visited = set() + sys.stdout.write(' File "%s", line %i, in %s\n' + % (self.co_filename.proxyval(visited), + self.current_line_num(), + self.co_name.proxyval(visited))) + class PySetObjectPtr(PyObjectPtr): _typename = 'PySetObject'

@@ -1222,6 +1252,23 @@ iter_frame = iter_frame.newer() return index

+ # We divide frames into: + # - "python frames": + # - "bytecode frames" i.e. PyEval_EvalFrameEx + # - "other python frames": things that are of interest from a python + # POV, but aren't bytecode (e.g. GC, GIL) + # - everything else + + def is_python_frame(self): + '''Is this a PyEval_EvalFrameEx frame, or some other important + frame? (see is_other_python_frame for what "important" means in this + context)''' + if self.is_evalframeex(): + return True + if self.is_other_python_frame(): + return True + return False + def is_evalframeex(self): '''Is this a PyEval_EvalFrameEx frame?''' if self._gdbframe.name() == 'PyEval_EvalFrameEx': @@ -1238,6 +1285,50 @@

return False

+ def is_other_python_frame(self): + '''Is this frame worth displaying in python backtraces? + Examples: + - waiting on the GIL + - garbage-collecting + - within a CFunction + If it is, return a descriptive string + For other frames, return False + ''' + if self.is_waiting_for_gil(): + return 'Waiting for the GIL' + elif self.is_gc_collect(): + return 'Garbage-collecting' + else: + # Detect invocations of PyCFunction instances: + older = self.older() + if older and older._gdbframe.name() == 'PyCFunction_Call': + # Within that frame: + # "func" is the local containing the PyObject* of the + # PyCFunctionObject instance + # "f" is the same value, but cast to (PyCFunctionObject*) + # "self" is the (PyObject*) of the 'self' + try: + # Use the prettyprinter for the func: + func = older._gdbframe.read_var('func') + return str(func) + except RuntimeError: + return 'PyCFunction invocation (unable to read "func")' + + # This frame isn't worth reporting: + return False + + def is_waiting_for_gil(self): + '''Is this frame waiting on the GIL?''' + # This assumes the _POSIX_THREADS version of Python/ceval_gil.h: + name = self._gdbframe.name() + if name: + return ('PyThread_acquire_lock' in name + and 'lock_PyThread_acquire_lock' not in name) + + def is_gc_collect(self): + '''Is this frame "collect" within the garbage-collector?''' + return self._gdbframe.name() == 'collect' + def get_pyop(self): try: f = self._gdbframe.read_var('f') @@ -1267,8 +1358,22 @@

@classmethod def get_selected_python_frame(cls): - '''Try to obtain the Frame for the python code in the selected frame, - or None''' + '''Try to obtain the Frame for the python-related code in the selected + frame, or None''' + frame = cls.get_selected_frame() + + while frame: + if frame.is_python_frame(): + return frame + frame = frame.older() + + # Not found: + return None + + @classmethod + def get_selected_bytecode_frame(cls): + '''Try to obtain the Frame for the python bytecode interpreter in the + selected GDB frame, or None''' frame = cls.get_selected_frame()

while frame: @@ -1283,14 +1388,38 @@ if self.is_evalframeex(): pyop = self.get_pyop() if pyop: - sys.stdout.write('#%i %s\n' % (self.get_index(), pyop.get_truncated_repr(MAX_OUTPUT_LEN))) + line = pyop.get_truncated_repr(MAX_OUTPUT_LEN) + write_unicode(sys.stdout, '#%i %s\n' % (self.get_index(), line)) if not pyop.is_optimized_out(): line = pyop.current_line() - sys.stdout.write(' %s\n' % line.strip()) + if line is not None: + sys.stdout.write(' %s\n' % line.strip()) else: sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) else: - sys.stdout.write('#%i\n' % self.get_index()) + info = self.is_other_python_frame() + if info: + sys.stdout.write('#%i %s\n' % (self.get_index(), info)) + else: + sys.stdout.write('#%i\n' % self.get_index()) + + def print_traceback(self): + if self.is_evalframeex(): + pyop = self.get_pyop() + if pyop: + pyop.print_traceback() + if not pyop.is_optimized_out(): + line = pyop.current_line() + if line is not None: + sys.stdout.write(' %s\n' % line.strip()) + else: + sys.stdout.write(' (unable to read python frame information)\n') + else: + info = self.is_other_python_frame() + if info: + sys.stdout.write(' %s\n' % info) + else: + sys.stdout.write(' (not a python frame)\n')

class PyList(gdb.Command): '''List the current Python source code, if any @@ -1326,9 +1455,10 @@ if m: start, end = map(int, m.groups())

- frame = Frame.get_selected_python_frame() + # py-list requires an actual PyEval_EvalFrameEx frame: + frame = Frame.get_selected_bytecode_frame() if not frame: - print('Unable to locate python frame') + print('Unable to locate gdb frame for python bytecode interpreter') return

pyop = frame.get_pyop() @@ -1346,7 +1476,13 @@ if start<1: start = 1

- with open(filename, 'r') as f: + try: + f = open(filename, 'r') + except IOError as err: + sys.stdout.write('Unable to open %s: %s\n' + % (filename, err)) + return + with f: all_lines = f.readlines() # start and end are 1-based, all_lines is 0-based; # so [start-1:end] as a python slice gives us [start, end] as a @@ -1374,7 +1510,7 @@ if not iter_frame: break

- if iter_frame.is_evalframeex(): + if iter_frame.is_python_frame(): # Result: if iter_frame.select(): iter_frame.print_summary() @@ -1416,6 +1552,24 @@ PyUp() PyDown()

+class PyBacktraceFull(gdb.Command): + 'Display the current python frame and all the frames within its call stack (if any)' + def __init__(self): + gdb.Command.__init__ (self, + "py-bt-full", + gdb.COMMAND_STACK, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + frame = Frame.get_selected_python_frame() + while frame: + if frame.is_python_frame(): + frame.print_summary() + frame = frame.older() + +PyBacktraceFull() + class PyBacktrace(gdb.Command): 'Display the current python frame and all the frames within its call stack (if any)' def __init__(self): @@ -1426,10 +1580,11 @@

def invoke(self, args, from_tty): + sys.stdout.write('Traceback (most recent call first):\n') frame = Frame.get_selected_python_frame() while frame: - if frame.is_evalframeex(): - frame.print_summary() + if frame.is_python_frame(): + frame.print_traceback() frame = frame.older()

PyBacktrace()

-- Repository URL: https://hg.python.org/cpython

_______________________________________________ Python-checkins mailing list Python-checkins@python.org https://mail.python.org/mailman/listinfo/python-checkins