Mailman 3 February 2018 - Python-checkins

[3.6] bpo-32922: dbm.open() now encodes filename with the filesystem encoding. (GH-5832). (GH-5906)
by Serhiy Storchaka Feb. 27, 2018

Feb. 27, 2018

https://github.com/python/cpython/commit/5bf74859fe253f0e8ff73ba699cd1a7e2d… commit: 5bf74859fe253f0e8ff73ba699cd1a7e2dfbcea3 branch: 3.6 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-02-27T03:39:53+02:00 summary: [3.6] bpo-32922: dbm.open() now encodes filename with the filesystem encoding. (GH-5832). (GH-5906) (cherry picked from commit 6f600ff1734ca2fdcdd37a809adf8130f0d8cc4e) Co-authored-by: Serhiy Storchaka <storchaka(… [View More]a)gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-23-19-12-04.bpo-32922.u-xe0B.rst M Lib/test/test_dbm_dumb.py M Lib/test/test_dbm_gnu.py M Lib/test/test_dbm_ndbm.py M Modules/_dbmmodule.c M Modules/_gdbmmodule.c M Modules/clinic/_dbmmodule.c.h M Modules/clinic/_gdbmmodule.c.h diff --git a/Lib/test/test_dbm_dumb.py b/Lib/test/test_dbm_dumb.py index df531d64e488..2606ba78eb55 100644 --- a/Lib/test/test_dbm_dumb.py +++ b/Lib/test/test_dbm_dumb.py @@ -275,6 +275,21 @@ def test_readonly_files(self): self.assertEqual(sorted(f.keys()), sorted(self._dict)) f.close() # don't write + @unittest.skipUnless(support.TESTFN_NONASCII, + 'requires OS support of non-ASCII encodings') + def test_nonascii_filename(self): + filename = support.TESTFN_NONASCII + for suffix in ['.dir', '.dat', '.bak']: + self.addCleanup(support.unlink, filename + suffix) + with dumbdbm.open(filename, 'c') as db: + db[b'key'] = b'value' + self.assertTrue(os.path.exists(filename + '.dat')) + self.assertTrue(os.path.exists(filename + '.dir')) + with dumbdbm.open(filename, 'r') as db: + self.assertEqual(list(db.keys()), [b'key']) + self.assertTrue(b'key' in db) + self.assertEqual(db[b'key'], b'value') + def tearDown(self): _delete_files() diff --git a/Lib/test/test_dbm_gnu.py b/Lib/test/test_dbm_gnu.py index 304b33286978..d96df9284806 100644 --- a/Lib/test/test_dbm_gnu.py +++ b/Lib/test/test_dbm_gnu.py @@ -2,7 +2,7 @@ gdbm = support.import_module("dbm.gnu") #skip if not supported import unittest import os -from test.support import TESTFN, unlink +from test.support import TESTFN, TESTFN_NONASCII, unlink filename = TESTFN @@ -93,5 +93,39 @@ def test_context_manager(self): self.assertEqual(str(cm.exception), "GDBM object has already been closed") + def test_bytes(self): + with gdbm.open(filename, 'c') as db: + db[b'bytes key \xbd'] = b'bytes value \xbd' + with gdbm.open(filename, 'r') as db: + self.assertEqual(list(db.keys()), [b'bytes key \xbd']) + self.assertTrue(b'bytes key \xbd' in db) + self.assertEqual(db[b'bytes key \xbd'], b'bytes value \xbd') + + def test_unicode(self): + with gdbm.open(filename, 'c') as db: + db['Unicode key \U0001f40d'] = 'Unicode value \U0001f40d' + with gdbm.open(filename, 'r') as db: + self.assertEqual(list(db.keys()), ['Unicode key \U0001f40d'.encode()]) + self.assertTrue('Unicode key \U0001f40d'.encode() in db) + self.assertTrue('Unicode key \U0001f40d' in db) + self.assertEqual(db['Unicode key \U0001f40d'.encode()], + 'Unicode value \U0001f40d'.encode()) + self.assertEqual(db['Unicode key \U0001f40d'], + 'Unicode value \U0001f40d'.encode()) + + @unittest.skipUnless(TESTFN_NONASCII, + 'requires OS support of non-ASCII encodings') + def test_nonascii_filename(self): + filename = TESTFN_NONASCII + self.addCleanup(unlink, filename) + with gdbm.open(filename, 'c') as db: + db[b'key'] = b'value' + self.assertTrue(os.path.exists(filename)) + with gdbm.open(filename, 'r') as db: + self.assertEqual(list(db.keys()), [b'key']) + self.assertTrue(b'key' in db) + self.assertEqual(db[b'key'], b'value') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_dbm_ndbm.py b/Lib/test/test_dbm_ndbm.py index 49f4426e4cb9..fb7d0e8281e3 100644 --- a/Lib/test/test_dbm_ndbm.py +++ b/Lib/test/test_dbm_ndbm.py @@ -1,5 +1,6 @@ from test import support support.import_module("dbm.ndbm") #skip if not supported +import os import unittest import dbm.ndbm from dbm.ndbm import error @@ -47,6 +48,42 @@ def test_context_manager(self): self.assertEqual(str(cm.exception), "DBM object has already been closed") + def test_bytes(self): + with dbm.ndbm.open(self.filename, 'c') as db: + db[b'bytes key \xbd'] = b'bytes value \xbd' + with dbm.ndbm.open(self.filename, 'r') as db: + self.assertEqual(list(db.keys()), [b'bytes key \xbd']) + self.assertTrue(b'bytes key \xbd' in db) + self.assertEqual(db[b'bytes key \xbd'], b'bytes value \xbd') + + def test_unicode(self): + with dbm.ndbm.open(self.filename, 'c') as db: + db['Unicode key \U0001f40d'] = 'Unicode value \U0001f40d' + with dbm.ndbm.open(self.filename, 'r') as db: + self.assertEqual(list(db.keys()), ['Unicode key \U0001f40d'.encode()]) + self.assertTrue('Unicode key \U0001f40d'.encode() in db) + self.assertTrue('Unicode key \U0001f40d' in db) + self.assertEqual(db['Unicode key \U0001f40d'.encode()], + 'Unicode value \U0001f40d'.encode()) + self.assertEqual(db['Unicode key \U0001f40d'], + 'Unicode value \U0001f40d'.encode()) + + @unittest.skipUnless(support.TESTFN_NONASCII, + 'requires OS support of non-ASCII encodings') + def test_nonascii_filename(self): + filename = support.TESTFN_NONASCII + for suffix in ['', '.pag', '.dir', '.db']: + self.addCleanup(support.unlink, filename + suffix) + with dbm.ndbm.open(filename, 'c') as db: + db[b'key'] = b'value' + self.assertTrue(any(os.path.exists(filename + suffix) + for suffix in ['', '.pag', '.dir', '.db'])) + with dbm.ndbm.open(filename, 'r') as db: + self.assertEqual(list(db.keys()), [b'key']) + self.assertTrue(b'key' in db) + self.assertEqual(db[b'key'], b'value') + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-02-23-19-12-04.bpo-32922.u-xe0B.rst b/Misc/NEWS.d/next/Library/2018-02-23-19-12-04.bpo-32922.u-xe0B.rst new file mode 100644 index 000000000000..412e588586c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-23-19-12-04.bpo-32922.u-xe0B.rst @@ -0,0 +1,2 @@ +dbm.open() now encodes filename with the filesystem encoding rather than +default encoding. diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 804978a6ac59..842953679698 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -413,7 +413,7 @@ static PyTypeObject Dbmtype = { _dbm.open as dbmopen - filename: str + filename: unicode The filename to open. flags: str="r" @@ -430,9 +430,9 @@ Return a database object. [clinic start generated code]*/ static PyObject * -dbmopen_impl(PyObject *module, const char *filename, const char *flags, +dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, int mode) -/*[clinic end generated code: output=5fade8cf16e0755f input=226334bade5764e6]*/ +/*[clinic end generated code: output=9527750f5df90764 input=376a9d903a50df59]*/ { int iflags; @@ -451,7 +451,20 @@ dbmopen_impl(PyObject *module, const char *filename, const char *flags, "arg 2 to open should be 'r', 'w', 'c', or 'n'"); return NULL; } - return newdbmobject(filename, iflags, mode); + + PyObject *filenamebytes = PyUnicode_EncodeFSDefault(filename); + if (filenamebytes == NULL) { + return NULL; + } + const char *name = PyBytes_AS_STRING(filenamebytes); + if (strlen(name) != (size_t)PyBytes_GET_SIZE(filenamebytes)) { + Py_DECREF(filenamebytes); + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + PyObject *self = newdbmobject(name, iflags, mode); + Py_DECREF(filenamebytes); + return self; } static PyMethodDef dbmmodule_methods[] = { diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index daae71dc8dca..a6ce35246483 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -532,7 +532,7 @@ static PyTypeObject Dbmtype = { /*[clinic input] _gdbm.open as dbmopen - filename as name: str + filename: unicode flags: str="r" mode: int(py_default="0o666") = 0o666 / @@ -562,8 +562,9 @@ when the database has to be created. It defaults to octal 0o666. [clinic start generated code]*/ static PyObject * -dbmopen_impl(PyObject *module, const char *name, const char *flags, int mode) -/*[clinic end generated code: output=31aa1bafdf5da688 input=55563cd60e51984a]*/ +dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, + int mode) +/*[clinic end generated code: output=9527750f5df90764 input=3be0b0875974b928]*/ { int iflags; @@ -611,7 +612,19 @@ dbmopen_impl(PyObject *module, const char *name, const char *flags, int mode) } } - return newdbmobject(name, iflags, mode); + PyObject *filenamebytes = PyUnicode_EncodeFSDefault(filename); + if (filenamebytes == NULL) { + return NULL; + } + const char *name = PyBytes_AS_STRING(filenamebytes); + if (strlen(name) != (size_t)PyBytes_GET_SIZE(filenamebytes)) { + Py_DECREF(filenamebytes); + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + PyObject *self = newdbmobject(name, iflags, mode); + Py_DECREF(filenamebytes); + return self; } static const char dbmmodule_open_flags[] = "rwcn" diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 06cf7e674114..7dd529a50c61 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -121,18 +121,18 @@ PyDoc_STRVAR(dbmopen__doc__, {"open", (PyCFunction)dbmopen, METH_VARARGS, dbmopen__doc__}, static PyObject * -dbmopen_impl(PyObject *module, const char *filename, const char *flags, +dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, int mode); static PyObject * dbmopen(PyObject *module, PyObject *args) { PyObject *return_value = NULL; - const char *filename; + PyObject *filename; const char *flags = "r"; int mode = 438; - if (!PyArg_ParseTuple(args, "s|si:open", + if (!PyArg_ParseTuple(args, "U|si:open", &filename, &flags, &mode)) { goto exit; } @@ -141,4 +141,4 @@ dbmopen(PyObject *module, PyObject *args) exit: return return_value; } -/*[clinic end generated code: output=99adf966ef0475ff input=a9049054013a1b77]*/ +/*[clinic end generated code: output=001fabffcecb99f1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index fdd589c07a4e..1030d7059e6f 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -234,23 +234,24 @@ PyDoc_STRVAR(dbmopen__doc__, {"open", (PyCFunction)dbmopen, METH_VARARGS, dbmopen__doc__}, static PyObject * -dbmopen_impl(PyObject *module, const char *name, const char *flags, int mode); +dbmopen_impl(PyObject *module, PyObject *filename, const char *flags, + int mode); static PyObject * dbmopen(PyObject *module, PyObject *args) { PyObject *return_value = NULL; - const char *name; + PyObject *filename; const char *flags = "r"; int mode = 438; - if (!PyArg_ParseTuple(args, "s|si:open", - &name, &flags, &mode)) { + if (!PyArg_ParseTuple(args, "U|si:open", + &filename, &flags, &mode)) { goto exit; } - return_value = dbmopen_impl(module, name, flags, mode); + return_value = dbmopen_impl(module, filename, flags, mode); exit: return return_value; } -/*[clinic end generated code: output=ed0f5d4e3d79b80c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=afb99364ac420d10 input=a9049054013a1b77]*/ [View Less]

1 0

bpo-32960: For dataclasses, disallow inheriting frozen from non-frozen classes and vice-versa, (GH-5919)
by Eric V. Smith Feb. 27, 2018

Feb. 27, 2018

https://github.com/python/cpython/commit/2fa6b9eae07e2385e2acbf2e40093a21fb… commit: 2fa6b9eae07e2385e2acbf2e40093a21fb3a10c4 branch: master author: Eric V. Smith <ericvsmith(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-02-26T20:38:33-05:00 summary: bpo-32960: For dataclasses, disallow inheriting frozen from non-frozen classes and vice-versa, (GH-5919) This restriction will be relaxed at a future date. files: A Misc/NEWS.d/next/Library/2018-02-… [View More]26-20-04-40.bpo-32960.48r0Ml.rst M Lib/dataclasses.py M Lib/test/test_dataclasses.py diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index db92b10960d5..54478fec93df 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -623,14 +623,21 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): else: setattr(cls, f.name, f.default) + # We're inheriting from a frozen dataclass, but we're not frozen. + if cls.__setattr__ is _frozen_setattr and not frozen: + raise TypeError('cannot inherit non-frozen dataclass from a ' + 'frozen one') + + # We're inheriting from a non-frozen dataclass, but we're frozen. + if (hasattr(cls, _MARKER) and cls.__setattr__ is not _frozen_setattr + and frozen): + raise TypeError('cannot inherit frozen dataclass from a ' + 'non-frozen one') + # Remember all of the fields on our class (including bases). This # marks this class as being a dataclass. setattr(cls, _MARKER, fields) - # We also need to check if a parent class is frozen: frozen has to - # be inherited down. - is_frozen = frozen or cls.__setattr__ is _frozen_setattr - # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically # set __hash__ to None. This is a heuristic, as it's possible @@ -654,7 +661,7 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): if f._field_type in (_FIELD, _FIELD_INITVAR)] _set_new_attribute(cls, '__init__', _init_fn(flds, - is_frozen, + frozen, has_post_init, # The name to use for the "self" param # in __init__. Use "self" if possible. @@ -696,7 +703,7 @@ def _process_class(cls, repr, eq, order, unsafe_hash, init, frozen): f'in class {cls.__name__}. Consider using ' 'functools.total_ordering') - if is_frozen: + if frozen: for name, fn in [('__setattr__', _frozen_setattr), ('__delattr__', _frozen_delattr)]: if _set_new_attribute(cls, name, fn): diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses.py index 582cb3459f5d..46d485c0157b 100755 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses.py @@ -637,29 +637,6 @@ class C: y: int self.assertNotEqual(Point(1, 3), C(1, 3)) - def test_frozen(self): - @dataclass(frozen=True) - class C: - i: int - - c = C(10) - self.assertEqual(c.i, 10) - with self.assertRaises(FrozenInstanceError): - c.i = 5 - self.assertEqual(c.i, 10) - - # Check that a derived class is still frozen, even if not - # marked so. - @dataclass - class D(C): - pass - - d = D(20) - self.assertEqual(d.i, 20) - with self.assertRaises(FrozenInstanceError): - d.i = 5 - self.assertEqual(d.i, 20) - def test_not_tuple(self): # Test that some of the problems with namedtuple don't happen # here. @@ -2475,5 +2452,66 @@ class C(base): assert False, f'unknown value for expected={expected!r}' +class TestFrozen(unittest.TestCase): + def test_frozen(self): + @dataclass(frozen=True) + class C: + i: int + + c = C(10) + self.assertEqual(c.i, 10) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertEqual(c.i, 10) + + def test_inherit(self): + @dataclass(frozen=True) + class C: + i: int + + @dataclass(frozen=True) + class D(C): + j: int + + d = D(0, 10) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + self.assertEqual(d.i, 0) + + def test_inherit_from_nonfrozen_from_frozen(self): + @dataclass(frozen=True) + class C: + i: int + + with self.assertRaisesRegex(TypeError, + 'cannot inherit non-frozen dataclass from a frozen one'): + @dataclass + class D(C): + pass + + def test_inherit_from_frozen_from_nonfrozen(self): + @dataclass + class C: + i: int + + with self.assertRaisesRegex(TypeError, + 'cannot inherit frozen dataclass from a non-frozen one'): + @dataclass(frozen=True) + class D(C): + pass + + def test_inherit_from_normal_class(self): + class C: + pass + + @dataclass(frozen=True) + class D(C): + i: int + + d = D(10) + with self.assertRaises(FrozenInstanceError): + d.i = 5 + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2018-02-26-20-04-40.bpo-32960.48r0Ml.rst b/Misc/NEWS.d/next/Library/2018-02-26-20-04-40.bpo-32960.48r0Ml.rst new file mode 100644 index 000000000000..4ad1fa17571d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-26-20-04-40.bpo-32960.48r0Ml.rst @@ -0,0 +1,3 @@ +For dataclasses, disallow inheriting frozen from non-frozen classes, and +also disallow inheriting non-frozen from frozen classes. This restriction +will be relaxed at a future date. [View Less]

1 0

bpo-32713: Fix tarfile.itn for large/negative float values. (GH-5434)
by Miss Islington (bot) Feb. 27, 2018

Feb. 27, 2018

https://github.com/python/cpython/commit/3f6c172b702dccc90d16db8faa14b4fabd… commit: 3f6c172b702dccc90d16db8faa14b4fabda63ff6 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-02-26T17:11:01-08:00 summary: bpo-32713: Fix tarfile.itn for large/negative float values. (GH-5434) (cherry picked from commit 72d9b2be36f091793ae7ffc5ad751f040c6e6ad3) Co-authored-by: Joffrey F <f.joffrey(a)… [View More]gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 631b69dcba4f..395b846ee2c4 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -202,8 +202,9 @@ def itn(n, digits=8, format=DEFAULT_FORMAT): # base-256 representation. This allows values up to (256**(digits-1))-1. # A 0o200 byte indicates a positive number, a 0o377 byte a negative # number. + n = int(n) if 0 <= n < 8 ** (digits - 1): - s = bytes("%0*o" % (digits - 1, int(n)), "ascii") + NUL + s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1): if n >= 0: s = bytearray([0o200]) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index fc7905542116..4cd7d5370f58 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -2146,6 +2146,14 @@ def test_write_number_fields(self): self.assertEqual(tarfile.itn(-0x100000000000000), b"\xff\x00\x00\x00\x00\x00\x00\x00") + # Issue 32713: Test if itn() supports float values outside the + # non-GNU format range + self.assertEqual(tarfile.itn(-100.0, format=tarfile.GNU_FORMAT), + b"\xff\xff\xff\xff\xff\xff\xff\x9c") + self.assertEqual(tarfile.itn(8 ** 12 + 0.0, format=tarfile.GNU_FORMAT), + b"\x80\x00\x00\x10\x00\x00\x00\x00") + self.assertEqual(tarfile.nti(tarfile.itn(-0.1, format=tarfile.GNU_FORMAT)), 0) + def test_number_field_limits(self): with self.assertRaises(ValueError): tarfile.itn(-1, 8, tarfile.USTAR_FORMAT) diff --git a/Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst b/Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst new file mode 100644 index 000000000000..bb5d64a351cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst @@ -0,0 +1 @@ +Fixed tarfile.itn handling of out-of-bounds float values. Patch by Joffrey Fuhrer. [View Less]

1 0

bpo-32713: Fix tarfile.itn for large/negative float values. (GH-5434)
by Miss Islington (bot) Feb. 27, 2018

Feb. 27, 2018

https://github.com/python/cpython/commit/a9a8a9814a52b3c92b3680f0b3a3561165… commit: a9a8a9814a52b3c92b3680f0b3a356116510ee18 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-02-26T16:50:09-08:00 summary: bpo-32713: Fix tarfile.itn for large/negative float values. (GH-5434) (cherry picked from commit 72d9b2be36f091793ae7ffc5ad751f040c6e6ad3) Co-authored-by: Joffrey F <f.joffrey(a)… [View More]gmail.com> files: A Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst M Lib/tarfile.py M Lib/test/test_tarfile.py diff --git a/Lib/tarfile.py b/Lib/tarfile.py index a24ee42abf82..85119a48a48b 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -200,8 +200,9 @@ def itn(n, digits=8, format=DEFAULT_FORMAT): # base-256 representation. This allows values up to (256**(digits-1))-1. # A 0o200 byte indicates a positive number, a 0o377 byte a negative # number. + n = int(n) if 0 <= n < 8 ** (digits - 1): - s = bytes("%0*o" % (digits - 1, int(n)), "ascii") + NUL + s = bytes("%0*o" % (digits - 1, n), "ascii") + NUL elif format == GNU_FORMAT and -256 ** (digits - 1) <= n < 256 ** (digits - 1): if n >= 0: s = bytearray([0o200]) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index b868326d5c74..7d2eec8a7ccf 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -2149,6 +2149,14 @@ def test_write_number_fields(self): self.assertEqual(tarfile.itn(-0x100000000000000), b"\xff\x00\x00\x00\x00\x00\x00\x00") + # Issue 32713: Test if itn() supports float values outside the + # non-GNU format range + self.assertEqual(tarfile.itn(-100.0, format=tarfile.GNU_FORMAT), + b"\xff\xff\xff\xff\xff\xff\xff\x9c") + self.assertEqual(tarfile.itn(8 ** 12 + 0.0, format=tarfile.GNU_FORMAT), + b"\x80\x00\x00\x10\x00\x00\x00\x00") + self.assertEqual(tarfile.nti(tarfile.itn(-0.1, format=tarfile.GNU_FORMAT)), 0) + def test_number_field_limits(self): with self.assertRaises(ValueError): tarfile.itn(-1, 8, tarfile.USTAR_FORMAT) diff --git a/Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst b/Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst new file mode 100644 index 000000000000..bb5d64a351cb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-02-26-13-16-36.bpo-32713.55yegW.rst @@ -0,0 +1 @@ +Fixed tarfile.itn handling of out-of-bounds float values. Patch by Joffrey Fuhrer. [View Less]

1 0

bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745)
by Miss Islington (bot) Feb. 27, 2018

Feb. 27, 2018

https://github.com/python/cpython/commit/ec5569b08efd28fdc032fb8b474f588633… commit: ec5569b08efd28fdc032fb8b474f588633ddf6af branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-02-26T15:59:55-08:00 summary: bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745) (cherry picked from commit eee72d4778a5513038edd5236cdd87ccce2bc60a) Co-authored-by: Tobotimus … [View More]<Tobotimus(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst M Lib/test/test_tools/test_i18n.py M Tools/i18n/pygettext.py diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index ce6f549879c7..5e7c4bffccde 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -2,6 +2,7 @@ import os import unittest +import textwrap from test.support.script_helper import assert_python_ok from test.test_tools import skip_if_missing, toolsdir @@ -27,6 +28,41 @@ def get_header(self, data): headers[key] = val.strip() return headers + def get_msgids(self, data): + """ utility: return all msgids in .po file as a list of strings """ + msgids = [] + reading_msgid = False + cur_msgid = [] + for line in data.split('\n'): + if reading_msgid: + if line.startswith('"'): + cur_msgid.append(line.strip('"')) + else: + msgids.append('\n'.join(cur_msgid)) + cur_msgid = [] + reading_msgid = False + continue + if line.startswith('msgid '): + line = line[len('msgid '):] + cur_msgid.append(line.strip('"')) + reading_msgid = True + else: + if reading_msgid: + msgids.append('\n'.join(cur_msgid)) + + return msgids + + def extract_docstrings_from_str(self, module_content): + """ utility: return all msgids extracted from module_content """ + filename = 'test_docstrings.py' + with temp_cwd(None) as cwd: + with open(filename, 'w') as fp: + fp.write(module_content) + assert_python_ok(self.script, '-D', filename) + with open('messages.pot') as fp: + data = fp.read() + return self.get_msgids(data) + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry @@ -70,3 +106,55 @@ def test_POT_Creation_Date(self): # This will raise if the date format does not exactly match. datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + + def test_funcdocstring_annotated_args(self): + """ Test docstrings for functions with annotated args """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar: str): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_annotated_return(self): + """ Test docstrings for functions with annotated return type """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar) -> str: + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_defvalue_args(self): + """ Test docstring for functions with default arg values """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar=()): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_multiple_funcs(self): + """ Test docstring extraction for multiple functions combining + annotated args, annotated return types and default arg values + """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo1(bar: tuple=()) -> str: + """doc1""" + + def foo2(bar: List[1:2]) -> (lambda x: x): + """doc2""" + + def foo3(bar: 'func'=lambda x: x) -> {1: 2}: + """doc3""" + ''')) + self.assertIn('doc1', msgids) + self.assertIn('doc2', msgids) + self.assertIn('doc3', msgids) + + def test_classdocstring_early_colon(self): + """ Test docstring extraction for a class with colons occuring within + the parentheses. + """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)): + """doc""" + ''')) + self.assertIn('doc', msgids) diff --git a/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst b/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst new file mode 100644 index 000000000000..b0b4c5e9357c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst @@ -0,0 +1,3 @@ +Fix pygettext not extracting docstrings for functions with type annotated +arguments. +Patch by Toby Harradine. diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 8ef5ff8a3e72..0f0395a4fcab 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -320,6 +320,7 @@ def __init__(self, options): self.__lineno = -1 self.__freshmodule = 1 self.__curfile = None + self.__enclosurecount = 0 def __call__(self, ttype, tstring, stup, etup, line): # dispatch @@ -340,7 +341,7 @@ def __waiting(self, ttype, tstring, lineno): elif ttype not in (tokenize.COMMENT, tokenize.NL): self.__freshmodule = 0 return - # class docstring? + # class or func/method docstring? if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suiteseen return @@ -348,9 +349,15 @@ def __waiting(self, ttype, tstring, lineno): self.__state = self.__keywordseen def __suiteseen(self, ttype, tstring, lineno): - # ignore anything until we see the colon - if ttype == tokenize.OP and tstring == ':': - self.__state = self.__suitedocstring + # skip over any enclosure pairs until we see the colon + if ttype == tokenize.OP: + if tstring == ':' and self.__enclosurecount == 0: + # we see a colon and we're not in an enclosure: end of def + self.__state = self.__suitedocstring + elif tstring in '([{': + self.__enclosurecount += 1 + elif tstring in ')]}': + self.__enclosurecount -= 1 def __suitedocstring(self, ttype, tstring, lineno): # ignore any intervening noise [View Less]

1 0

bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745)
by Miss Islington (bot) Feb. 26, 2018

Feb. 26, 2018

https://github.com/python/cpython/commit/51d95ffc2fb3337dc87277c9af25ecc9a0… commit: 51d95ffc2fb3337dc87277c9af25ecc9a01f991b branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-02-26T15:23:26-08:00 summary: bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745) (cherry picked from commit eee72d4778a5513038edd5236cdd87ccce2bc60a) Co-authored-by: Tobotimus … [View More]<Tobotimus(a)users.noreply.github.com> files: A Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst M Lib/test/test_tools/test_i18n.py M Tools/i18n/pygettext.py diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 5c28baea7241..3c8be27a57b4 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -3,6 +3,7 @@ import os import sys import unittest +import textwrap from test.support.script_helper import assert_python_ok from test.test_tools import skip_if_missing, toolsdir @@ -28,6 +29,41 @@ def get_header(self, data): headers[key] = val.strip() return headers + def get_msgids(self, data): + """ utility: return all msgids in .po file as a list of strings """ + msgids = [] + reading_msgid = False + cur_msgid = [] + for line in data.split('\n'): + if reading_msgid: + if line.startswith('"'): + cur_msgid.append(line.strip('"')) + else: + msgids.append('\n'.join(cur_msgid)) + cur_msgid = [] + reading_msgid = False + continue + if line.startswith('msgid '): + line = line[len('msgid '):] + cur_msgid.append(line.strip('"')) + reading_msgid = True + else: + if reading_msgid: + msgids.append('\n'.join(cur_msgid)) + + return msgids + + def extract_docstrings_from_str(self, module_content): + """ utility: return all msgids extracted from module_content """ + filename = 'test_docstrings.py' + with temp_cwd(None) as cwd: + with open(filename, 'w') as fp: + fp.write(module_content) + assert_python_ok(self.script, '-D', filename) + with open('messages.pot') as fp: + data = fp.read() + return self.get_msgids(data) + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry @@ -72,3 +108,55 @@ def test_POT_Creation_Date(self): # This will raise if the date format does not exactly match. datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + + def test_funcdocstring_annotated_args(self): + """ Test docstrings for functions with annotated args """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar: str): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_annotated_return(self): + """ Test docstrings for functions with annotated return type """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar) -> str: + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_defvalue_args(self): + """ Test docstring for functions with default arg values """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar=()): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_multiple_funcs(self): + """ Test docstring extraction for multiple functions combining + annotated args, annotated return types and default arg values + """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo1(bar: tuple=()) -> str: + """doc1""" + + def foo2(bar: List[1:2]) -> (lambda x: x): + """doc2""" + + def foo3(bar: 'func'=lambda x: x) -> {1: 2}: + """doc3""" + ''')) + self.assertIn('doc1', msgids) + self.assertIn('doc2', msgids) + self.assertIn('doc3', msgids) + + def test_classdocstring_early_colon(self): + """ Test docstring extraction for a class with colons occuring within + the parentheses. + """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)): + """doc""" + ''')) + self.assertIn('doc', msgids) diff --git a/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst b/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst new file mode 100644 index 000000000000..b0b4c5e9357c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst @@ -0,0 +1,3 @@ +Fix pygettext not extracting docstrings for functions with type annotated +arguments. +Patch by Toby Harradine. diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 8ef5ff8a3e72..0f0395a4fcab 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -320,6 +320,7 @@ def __init__(self, options): self.__lineno = -1 self.__freshmodule = 1 self.__curfile = None + self.__enclosurecount = 0 def __call__(self, ttype, tstring, stup, etup, line): # dispatch @@ -340,7 +341,7 @@ def __waiting(self, ttype, tstring, lineno): elif ttype not in (tokenize.COMMENT, tokenize.NL): self.__freshmodule = 0 return - # class docstring? + # class or func/method docstring? if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suiteseen return @@ -348,9 +349,15 @@ def __waiting(self, ttype, tstring, lineno): self.__state = self.__keywordseen def __suiteseen(self, ttype, tstring, lineno): - # ignore anything until we see the colon - if ttype == tokenize.OP and tstring == ':': - self.__state = self.__suitedocstring + # skip over any enclosure pairs until we see the colon + if ttype == tokenize.OP: + if tstring == ':' and self.__enclosurecount == 0: + # we see a colon and we're not in an enclosure: end of def + self.__state = self.__suitedocstring + elif tstring in '([{': + self.__enclosurecount += 1 + elif tstring in ')]}': + self.__enclosurecount -= 1 def __suitedocstring(self, ttype, tstring, lineno): # ignore any intervening noise [View Less]

1 0

bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745)
by Serhiy Storchaka Feb. 26, 2018

Feb. 26, 2018

https://github.com/python/cpython/commit/eee72d4778a5513038edd5236cdd87ccce… commit: eee72d4778a5513038edd5236cdd87ccce2bc60a branch: master author: Tobotimus <Tobotimus(a)users.noreply.github.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2018-02-27T00:48:14+02:00 summary: bpo-32222: Fix pygettext skipping docstrings for funcs with arg typehints (GH-4745) files: A Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst M Lib/test/test_tools/test_i18n.… [View More]py M Tools/i18n/pygettext.py diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 5c28baea7241..3c8be27a57b4 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -3,6 +3,7 @@ import os import sys import unittest +import textwrap from test.support.script_helper import assert_python_ok from test.test_tools import skip_if_missing, toolsdir @@ -28,6 +29,41 @@ def get_header(self, data): headers[key] = val.strip() return headers + def get_msgids(self, data): + """ utility: return all msgids in .po file as a list of strings """ + msgids = [] + reading_msgid = False + cur_msgid = [] + for line in data.split('\n'): + if reading_msgid: + if line.startswith('"'): + cur_msgid.append(line.strip('"')) + else: + msgids.append('\n'.join(cur_msgid)) + cur_msgid = [] + reading_msgid = False + continue + if line.startswith('msgid '): + line = line[len('msgid '):] + cur_msgid.append(line.strip('"')) + reading_msgid = True + else: + if reading_msgid: + msgids.append('\n'.join(cur_msgid)) + + return msgids + + def extract_docstrings_from_str(self, module_content): + """ utility: return all msgids extracted from module_content """ + filename = 'test_docstrings.py' + with temp_cwd(None) as cwd: + with open(filename, 'w') as fp: + fp.write(module_content) + assert_python_ok(self.script, '-D', filename) + with open('messages.pot') as fp: + data = fp.read() + return self.get_msgids(data) + def test_header(self): """Make sure the required fields are in the header, according to: http://www.gnu.org/software/gettext/manual/gettext.html#Header-Entry @@ -72,3 +108,55 @@ def test_POT_Creation_Date(self): # This will raise if the date format does not exactly match. datetime.strptime(creationDate, '%Y-%m-%d %H:%M%z') + + def test_funcdocstring_annotated_args(self): + """ Test docstrings for functions with annotated args """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar: str): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_annotated_return(self): + """ Test docstrings for functions with annotated return type """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar) -> str: + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_defvalue_args(self): + """ Test docstring for functions with default arg values """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo(bar=()): + """doc""" + ''')) + self.assertIn('doc', msgids) + + def test_funcdocstring_multiple_funcs(self): + """ Test docstring extraction for multiple functions combining + annotated args, annotated return types and default arg values + """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + def foo1(bar: tuple=()) -> str: + """doc1""" + + def foo2(bar: List[1:2]) -> (lambda x: x): + """doc2""" + + def foo3(bar: 'func'=lambda x: x) -> {1: 2}: + """doc3""" + ''')) + self.assertIn('doc1', msgids) + self.assertIn('doc2', msgids) + self.assertIn('doc3', msgids) + + def test_classdocstring_early_colon(self): + """ Test docstring extraction for a class with colons occuring within + the parentheses. + """ + msgids = self.extract_docstrings_from_str(textwrap.dedent('''\ + class D(L[1:2], F({1: 2}), metaclass=M(lambda x: x)): + """doc""" + ''')) + self.assertIn('doc', msgids) diff --git a/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst b/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst new file mode 100644 index 000000000000..b0b4c5e9357c --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2017-12-07-20-51-20.bpo-32222.hPBcGT.rst @@ -0,0 +1,3 @@ +Fix pygettext not extracting docstrings for functions with type annotated +arguments. +Patch by Toby Harradine. diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index 8ef5ff8a3e72..0f0395a4fcab 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -320,6 +320,7 @@ def __init__(self, options): self.__lineno = -1 self.__freshmodule = 1 self.__curfile = None + self.__enclosurecount = 0 def __call__(self, ttype, tstring, stup, etup, line): # dispatch @@ -340,7 +341,7 @@ def __waiting(self, ttype, tstring, lineno): elif ttype not in (tokenize.COMMENT, tokenize.NL): self.__freshmodule = 0 return - # class docstring? + # class or func/method docstring? if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suiteseen return @@ -348,9 +349,15 @@ def __waiting(self, ttype, tstring, lineno): self.__state = self.__keywordseen def __suiteseen(self, ttype, tstring, lineno): - # ignore anything until we see the colon - if ttype == tokenize.OP and tstring == ':': - self.__state = self.__suitedocstring + # skip over any enclosure pairs until we see the colon + if ttype == tokenize.OP: + if tstring == ':' and self.__enclosurecount == 0: + # we see a colon and we're not in an enclosure: end of def + self.__state = self.__suitedocstring + elif tstring in '([{': + self.__enclosurecount += 1 + elif tstring in ')]}': + self.__enclosurecount -= 1 def __suitedocstring(self, ttype, tstring, lineno): # ignore any intervening noise [View Less]

1 0

bpo-18533: Avoid RuntimeError from repr() of recursive dictview (#4823) (#5357)
by Serhiy Storchaka Feb. 26, 2018

Feb. 26, 2018

https://github.com/python/cpython/commit/c20c97f6122e7a99e46cbd6c9ac8cb2941… commit: c20c97f6122e7a99e46cbd6c9ac8cb2941a3bf7a branch: 2.7 author: bennorth <ben(a)redfrontdoor.org> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2018-02-27T00:35:03+02:00 summary: bpo-18533: Avoid RuntimeError from repr() of recursive dictview (#4823) (#5357) (cherry picked from commit d7773d92bd11640a8c950d6c36a9cef1cee36f96) files: A Misc/NEWS.d/next/Core and Builtins/2017-12-13-16-46-… [View More]23.bpo-18533.Dlk8d7.rst M Lib/test/test_dictviews.py M Lib/test/test_ordered_dict.py M Objects/dictobject.c diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index b585bdd90203..1edeec55d077 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -1,5 +1,6 @@ import copy import pickle +import sys import unittest import collections from test import test_support @@ -169,6 +170,20 @@ def test_items_set_operations(self): def test_recursive_repr(self): d = {} d[42] = d.viewvalues() + r = repr(d) + # Cannot perform a stronger test, as the contents of the repr + # are implementation-dependent. All we can say is that we + # want a str result, not an exception of any sort. + self.assertIsInstance(r, str) + d[42] = d.viewitems() + r = repr(d) + # Again. + self.assertIsInstance(r, str) + + def test_deeply_nested_repr(self): + d = {} + for i in range(sys.getrecursionlimit() + 100): + d = {42: d.viewvalues()} self.assertRaises(RuntimeError, repr, d) def test_abc_registry(self): diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 85e4841fbb4b..17326c5190c6 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -220,6 +220,19 @@ def test_repr_recursive(self): self.assertEqual(repr(od), "OrderedDict([('a', None), ('b', None), ('c', None), ('x', ...)])") + def test_repr_recursive_values(self): + od = OrderedDict() + od[42] = od.viewvalues() + r = repr(od) + # Cannot perform a stronger test, as the contents of the repr + # are implementation-dependent. All we can say is that we + # want a str result, not an exception of any sort. + self.assertIsInstance(r, str) + od[42] = od.viewitems() + r = repr(od) + # Again. + self.assertIsInstance(r, str) + def test_setdefault(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) diff --git a/Misc/NEWS.d/next/Core and Builtins/2017-12-13-16-46-23.bpo-18533.Dlk8d7.rst b/Misc/NEWS.d/next/Core and Builtins/2017-12-13-16-46-23.bpo-18533.Dlk8d7.rst new file mode 100644 index 000000000000..2ffd5718d6ac --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2017-12-13-16-46-23.bpo-18533.Dlk8d7.rst @@ -0,0 +1,3 @@ +``repr()`` on a dict containing its own ``viewvalues()`` or +``viewitems()`` no longer raises ``RuntimeError``. Instead, use +``...``, as for other recursive structures. Patch by Ben North. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index a792b2dfa210..c544ecd8c2d2 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -3005,21 +3005,29 @@ dictview_repr(dictviewobject *dv) { PyObject *seq; PyObject *seq_str; - PyObject *result; + PyObject *result = NULL; + Py_ssize_t rc; + rc = Py_ReprEnter((PyObject *)dv); + if (rc != 0) { + return rc > 0 ? PyString_FromString("...") : NULL; + } seq = PySequence_List((PyObject *)dv); - if (seq == NULL) - return NULL; - + if (seq == NULL) { + goto Done; + } seq_str = PyObject_Repr(seq); + Py_DECREF(seq); + if (seq_str == NULL) { - Py_DECREF(seq); - return NULL; + goto Done; } result = PyString_FromFormat("%s(%s)", Py_TYPE(dv)->tp_name, PyString_AS_STRING(seq_str)); Py_DECREF(seq_str); - Py_DECREF(seq); + +Done: + Py_ReprLeave((PyObject *)dv); return result; } [View Less]

1 0

bpo-32836: Remove obsolete code from symtable pass (GH-5680)
by Nick Coghlan Feb. 26, 2018

Feb. 26, 2018

https://github.com/python/cpython/commit/3a087beddd9f0955eb9080a6fd1499ff89… commit: 3a087beddd9f0955eb9080a6fd1499ff89ca74bf branch: master author: Nitish Chandra <nitishchandrachinta(a)gmail.com> committer: Nick Coghlan <ncoghlan(a)gmail.com> date: 2018-02-27T07:31:20+10:00 summary: bpo-32836: Remove obsolete code from symtable pass (GH-5680) When comprehensions switched to using a nested scope, the old code for generating a temporary name to hold the accumulation target became … [View More]redundant, but was never actually removed. Patch by Nitish Chandra. files: A Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst M Include/symtable.h M Python/symtable.c diff --git a/Include/symtable.h b/Include/symtable.h index 86ae3c28e878..007f88db40e7 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -60,7 +60,6 @@ typedef struct _symtable_entry { int ste_col_offset; /* offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ - int ste_tmpname; /* counter for listcomp temp vars */ struct symtable *ste_table; } PySTEntryObject; diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst new file mode 100644 index 000000000000..4eeb9aa2e52c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-02-14-12-35-47.bpo-32836.bThJnx.rst @@ -0,0 +1 @@ +Don't use temporary variables in cases of list/dict/set comprehensions diff --git a/Python/symtable.c b/Python/symtable.c index ac14058fefd2..3e8c6f5dae30 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -69,7 +69,6 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_opt_col_offset = 0; - ste->ste_tmpname = 0; ste->ste_lineno = lineno; ste->ste_col_offset = col_offset; @@ -1082,24 +1081,6 @@ symtable_add_def(struct symtable *st, PyObject *name, int flag) } \ } -static int -symtable_new_tmpname(struct symtable *st) -{ - char tmpname[256]; - identifier tmp; - - PyOS_snprintf(tmpname, sizeof(tmpname), "_[%d]", - ++st->st_cur->ste_tmpname); - tmp = PyUnicode_InternFromString(tmpname); - if (!tmp) - return 0; - if (!symtable_add_def(st, tmp, DEF_LOCAL)) - return 0; - Py_DECREF(tmp); - return 1; -} - - static int symtable_record_directive(struct symtable *st, identifier name, stmt_ty s) { @@ -1723,7 +1704,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, expr_ty elt, expr_ty value) { int is_generator = (e->kind == GeneratorExp_kind); - int needs_tmp = !is_generator; comprehension_ty outermost = ((comprehension_ty) asdl_seq_GET(generators, 0)); /* Outermost iterator is evaluated in current scope */ @@ -1742,11 +1722,6 @@ symtable_handle_comprehension(struct symtable *st, expr_ty e, symtable_exit_block(st, (void *)e); return 0; } - /* Allocate temporary name if needed */ - if (needs_tmp && !symtable_new_tmpname(st)) { - symtable_exit_block(st, (void *)e); - return 0; - } VISIT(st, expr, outermost->target); VISIT_SEQ(st, expr, outermost->ifs); VISIT_SEQ_TAIL(st, comprehension, generators, 1); [View Less]

1 0

bpo-32147: Improved perfomance of binascii.unhexlify(). (GH-4586)
by Serhiy Storchaka Feb. 26, 2018

Feb. 26, 2018

https://github.com/python/cpython/commit/6b5df906afe113dbe421d044322254cfd4… commit: 6b5df906afe113dbe421d044322254cfd4747c9c branch: master author: Sergey Fedoseev <fedoseev.sergey(a)gmail.com> committer: Serhiy Storchaka <storchaka(a)gmail.com> date: 2018-02-26T22:35:41+02:00 summary: bpo-32147: Improved perfomance of binascii.unhexlify(). (GH-4586) files: A Misc/NEWS.d/next/Library/2017-11-28-10-23-13.bpo-32147.PI2k1Y.rst M Lib/test/test_binascii.py M Misc/ACKS M Modules/… [View More]binascii.c diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 8fa57cdf1b0b..0997d9432bf6 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -198,6 +198,11 @@ def test_hex(self): self.assertEqual(s, u) self.assertRaises(binascii.Error, binascii.a2b_hex, t[:-1]) self.assertRaises(binascii.Error, binascii.a2b_hex, t[:-1] + b'q') + self.assertRaises(binascii.Error, binascii.a2b_hex, bytes([255, 255])) + self.assertRaises(binascii.Error, binascii.a2b_hex, b'0G') + self.assertRaises(binascii.Error, binascii.a2b_hex, b'0g') + self.assertRaises(binascii.Error, binascii.a2b_hex, b'G0') + self.assertRaises(binascii.Error, binascii.a2b_hex, b'g0') # Confirm that b2a_hex == hexlify and a2b_hex == unhexlify self.assertEqual(binascii.hexlify(self.type2test(s)), t) diff --git a/Misc/ACKS b/Misc/ACKS index dee022f2ff0a..e2addfc210b7 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -460,6 +460,7 @@ Michael Farrell Troy J. Farrell Jim Fasarakis-Hilliard Mark Favas +Sergey Fedoseev Boris Feld Thomas Fenzl Niels Ferguson diff --git a/Misc/NEWS.d/next/Library/2017-11-28-10-23-13.bpo-32147.PI2k1Y.rst b/Misc/NEWS.d/next/Library/2017-11-28-10-23-13.bpo-32147.PI2k1Y.rst new file mode 100644 index 000000000000..e02a97c5e9e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-11-28-10-23-13.bpo-32147.PI2k1Y.rst @@ -0,0 +1,2 @@ +:func:`binascii.unhexlify` is now up to 2 times faster. +Patch by Sergey Fedoseev. diff --git a/Modules/binascii.c b/Modules/binascii.c index 1af6b7f98f25..59e99282ae35 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1130,21 +1130,6 @@ binascii_hexlify_impl(PyObject *module, Py_buffer *data) return _Py_strhex_bytes((const char *)data->buf, data->len); } -static int -to_int(int c) -{ - if (Py_ISDIGIT(c)) - return c - '0'; - else { - if (Py_ISUPPER(c)) - c = Py_TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} - - /*[clinic input] binascii.a2b_hex @@ -1187,9 +1172,9 @@ binascii_a2b_hex_impl(PyObject *module, Py_buffer *hexstr) retbuf = PyBytes_AS_STRING(retval); for (i=j=0; i < arglen; i += 2) { - int top = to_int(Py_CHARMASK(argbuf[i])); - int bot = to_int(Py_CHARMASK(argbuf[i+1])); - if (top == -1 || bot == -1) { + unsigned int top = _PyLong_DigitValue[Py_CHARMASK(argbuf[i])]; + unsigned int bot = _PyLong_DigitValue[Py_CHARMASK(argbuf[i+1])]; + if (top >= 16 || bot >= 16) { PyErr_SetString(Error, "Non-hexadecimal digit found"); goto finally; @@ -1218,19 +1203,6 @@ binascii_unhexlify_impl(PyObject *module, Py_buffer *hexstr) return binascii_a2b_hex_impl(module, hexstr); } -static const int table_hex[128] = { - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,-1,-1, -1,-1,-1,-1, - -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,10,11,12, 13,14,15,-1, -1,-1,-1,-1, -1,-1,-1,-1, - -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1 -}; - -#define hexval(c) table_hex[(unsigned int)(c)] - #define MAXLINESIZE 76 @@ -1293,9 +1265,9 @@ binascii_a2b_qp_impl(PyObject *module, Py_buffer *data, int header) (ascii_data[in+1] >= 'a' && ascii_data[in+1] <= 'f') || (ascii_data[in+1] >= '0' && ascii_data[in+1] <= '9'))) { /* hexval */ - ch = hexval(ascii_data[in]) << 4; + ch = _PyLong_DigitValue[ascii_data[in]] << 4; in++; - ch |= hexval(ascii_data[in]); + ch |= _PyLong_DigitValue[ascii_data[in]]; in++; odata[out++] = ch; } [View Less]

1 0