Mailman 3 July 2018 - Python-checkins

bpo-34113: Fix a crash when using LLTRACE is on (GH-8517)
by Victor Stinner 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/8ed317f1ca42a43df14282bbc3ccc0b561… commit: 8ed317f1ca42a43df14282bbc3ccc0b5610432f4 branch: master author: costypetrisor <costypetrisor(a)users.noreply.github.com> committer: Victor Stinner <vstinner(a)redhat.com> date: 2018-07-31T22:55:14+02:00 summary: bpo-34113: Fix a crash when using LLTRACE is on (GH-8517) Fix a crash on negative STACKADJ() when Low-Level trace (LLTRACE) is enabled. files: A Lib/test/test_lltrace.py A Misc/NEWS.d/next/Core and Builtins/2018-07-28-10-34-00.bpo-34113.eZ5FWV.rst M Python/ceval.c diff --git a/Lib/test/test_lltrace.py b/Lib/test/test_lltrace.py new file mode 100644 index 000000000000..49fae81eefab --- /dev/null +++ b/Lib/test/test_lltrace.py @@ -0,0 +1,31 @@ +import os +import textwrap +import unittest + +from test import support +from test.support.script_helper import assert_python_ok + + +class TestLLTrace(unittest.TestCase): + + def test_lltrace_does_not_crash_on_subscript_operator(self): + # If this test fails, it will reproduce a crash reported as + # bpo-34113. The crash happened at the command line console of + # debug Python builds with __ltrace__ enabled (only possible in console), + # when the interal Python stack was negatively adjusted + with open(support.TESTFN, 'w') as fd: + self.addCleanup(os.unlink, support.TESTFN) + fd.write(textwrap.dedent("""\ + import code + + console = code.InteractiveConsole() + console.push('__ltrace__ = 1') + console.push('a = [1, 2, 3]') + console.push('a[0] = 1') + print('unreachable if bug exists') + """)) + + assert_python_ok(support.TESTFN) + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-07-28-10-34-00.bpo-34113.eZ5FWV.rst b/Misc/NEWS.d/next/Core and Builtins/2018-07-28-10-34-00.bpo-34113.eZ5FWV.rst new file mode 100644 index 000000000000..f4c80f9b8a97 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-07-28-10-34-00.bpo-34113.eZ5FWV.rst @@ -0,0 +1,2 @@ +Fixed crash on debug builds when opcode stack was adjusted with negative +numbers. Patch by Constantin Petrisor. diff --git a/Python/ceval.c b/Python/ceval.c index 8f0e0e00c2b9..46da295aac06 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -762,16 +762,26 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) assert(STACK_LEVEL() <= co->co_stacksize); } #define POP() ((void)(lltrace && prtrace(TOP(), "pop")), \ BASIC_POP()) -#define STACKADJ(n) { (void)(BASIC_STACKADJ(n), \ +#define STACK_GROW(n) do { \ + assert(n >= 0); \ + (void)(BASIC_STACKADJ(n), \ lltrace && prtrace(TOP(), "stackadj")); \ - assert(STACK_LEVEL() <= co->co_stacksize); } + assert(STACK_LEVEL() <= co->co_stacksize); \ + } while (0) +#define STACK_SHRINK(n) do { \ + assert(n >= 0); \ + (void)(lltrace && prtrace(TOP(), "stackadj")); \ + (void)(BASIC_STACKADJ(-n)); \ + assert(STACK_LEVEL() <= co->co_stacksize); \ + } while (0) #define EXT_POP(STACK_POINTER) ((void)(lltrace && \ prtrace((STACK_POINTER)[-1], "ext_pop")), \ *--(STACK_POINTER)) #else #define PUSH(v) BASIC_PUSH(v) #define POP() BASIC_POP() -#define STACKADJ(n) BASIC_STACKADJ(n) +#define STACK_GROW(n) BASIC_STACKADJ(n) +#define STACK_SHRINK(n) BASIC_STACKADJ(-n) #define EXT_POP(STACK_POINTER) (*--(STACK_POINTER)) #endif @@ -1133,7 +1143,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *second = SECOND(); Py_INCREF(top); Py_INCREF(second); - STACKADJ(2); + STACK_GROW(2); SET_TOP(top); SET_SECOND(second); FAST_DISPATCH(); @@ -1173,7 +1183,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) SET_TOP(Py_False); DISPATCH(); } - STACKADJ(-1); + STACK_SHRINK(1); goto error; } @@ -1569,7 +1579,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *container = SECOND(); PyObject *v = THIRD(); int err; - STACKADJ(-3); + STACK_SHRINK(3); /* container[sub] = v */ err = PyObject_SetItem(container, sub, v); Py_DECREF(v); @@ -1584,7 +1594,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *sub = TOP(); PyObject *container = SECOND(); int err; - STACKADJ(-2); + STACK_SHRINK(2); /* del container[sub] */ err = PyObject_DelItem(container, sub); Py_DECREF(container); @@ -2067,7 +2077,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } } else if (unpack_iterable(seq, oparg, -1, stack_pointer + oparg)) { - STACKADJ(oparg); + STACK_GROW(oparg); } else { /* unpack_iterable() raised an exception */ Py_DECREF(seq); @@ -2097,7 +2107,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *owner = TOP(); PyObject *v = SECOND(); int err; - STACKADJ(-2); + STACK_SHRINK(2); err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); Py_DECREF(owner); @@ -2420,7 +2430,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) err = PySet_Add(set, item); Py_DECREF(item); } - STACKADJ(-oparg); + STACK_SHRINK(oparg); if (err != 0) { Py_DECREF(set); goto error; @@ -2641,7 +2651,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *value = SECOND(); PyObject *map; int err; - STACKADJ(-2); + STACK_SHRINK(2); map = PEEK(oparg); /* dict */ assert(PyDict_CheckExact(map)); err = PyDict_SetItem(map, key, value); /* map[key] = value */ @@ -2784,7 +2794,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *cond = TOP(); int err; if (cond == Py_True) { - STACKADJ(-1); + STACK_SHRINK(1); Py_DECREF(cond); FAST_DISPATCH(); } @@ -2794,7 +2804,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) } err = PyObject_IsTrue(cond); if (err > 0) { - STACKADJ(-1); + STACK_SHRINK(1); Py_DECREF(cond); } else if (err == 0) @@ -2808,7 +2818,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyObject *cond = TOP(); int err; if (cond == Py_False) { - STACKADJ(-1); + STACK_SHRINK(1); Py_DECREF(cond); FAST_DISPATCH(); } @@ -2821,7 +2831,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) JUMPTO(oparg); } else if (err == 0) { - STACKADJ(-1); + STACK_SHRINK(1); Py_DECREF(cond); } else @@ -2907,7 +2917,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) PyErr_Clear(); } /* iterator ended normally */ - STACKADJ(-1); + STACK_SHRINK(1); Py_DECREF(iter); JUMPBY(oparg); PREDICT(POP_BLOCK); @@ -3015,7 +3025,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) val = tb = Py_None; exc = TOP(); if (exc == NULL) { - STACKADJ(-1); + STACK_SHRINK(1); exit_func = TOP(); SET_TOP(exc); exc = Py_None;

1 0

[3.7] bpo-34263 Cap timeout submitted to epoll/select etc. to one day. (GH-8532) (GH-8586)
by Yury Selivanov 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/172a81e42bc30da1bd4027db9cd3b61724… commit: 172a81e42bc30da1bd4027db9cd3b6172469f7fe branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Yury Selivanov <yury(a)magic.io> date: 2018-07-31T11:29:07-04:00 summary: [3.7] bpo-34263 Cap timeout submitted to epoll/select etc. to one day. (GH-8532) (GH-8586) files: A Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst M Lib/asyncio/base_events.py diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index dc0ca3f02b9b..a79e123e5113 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -63,6 +63,9 @@ _HAS_IPv6 = hasattr(socket, 'AF_INET6') +# Maximum timeout passed to select to avoid OS limitations +MAXIMUM_SELECT_TIMEOUT = 24 * 3600 + def _format_handle(handle): cb = handle._callback @@ -1702,7 +1705,7 @@ def _run_once(self): elif self._scheduled: # Compute the desired timeout. when = self._scheduled[0]._when - timeout = max(0, when - self.time()) + timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT) if self._debug and timeout != 0: t0 = self.time() diff --git a/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst b/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst new file mode 100644 index 000000000000..799463b59163 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst @@ -0,0 +1,2 @@ +asyncio's event loop will not pass timeouts longer than one day to +epoll/select etc.

1 0

[3.6] bpo-34263 Cap timeout submitted to epoll/select etc. to one day. (GH-8532) (GH-8587)
by Yury Selivanov 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/6f16ffc1879fc934eba297b3e81bd940e3… commit: 6f16ffc1879fc934eba297b3e81bd940e32a7e03 branch: 3.6 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: Yury Selivanov <yury(a)magic.io> date: 2018-07-31T11:28:56-04:00 summary: [3.6] bpo-34263 Cap timeout submitted to epoll/select etc. to one day. (GH-8532) (GH-8587) files: A Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst M Lib/asyncio/base_events.py diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 79f0d63c5726..30ac5924d06b 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -56,6 +56,9 @@ _HAS_IPv6 = hasattr(socket, 'AF_INET6') +# Maximum timeout passed to select to avoid OS limitations +MAXIMUM_SELECT_TIMEOUT = 24 * 3600 + def _format_handle(handle): cb = handle._callback @@ -1378,7 +1381,7 @@ def _run_once(self): elif self._scheduled: # Compute the desired timeout. when = self._scheduled[0]._when - timeout = max(0, when - self.time()) + timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT) if self._debug and timeout != 0: t0 = self.time() diff --git a/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst b/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst new file mode 100644 index 000000000000..799463b59163 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst @@ -0,0 +1,2 @@ +asyncio's event loop will not pass timeouts longer than one day to +epoll/select etc.

1 0

bpo-34263 Cap timeout submitted to epoll/select etc. to one day. (GH-8532)
by Yury Selivanov 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/944451cd8d3e897138f4b43569de13cd08… commit: 944451cd8d3e897138f4b43569de13cd081ee251 branch: master author: MartinAltmayer <martin.altmayer(a)web.de> committer: Yury Selivanov <yury(a)magic.io> date: 2018-07-31T10:06:12-04:00 summary: bpo-34263 Cap timeout submitted to epoll/select etc. to one day. (GH-8532) files: A Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst M Doc/library/asyncio-eventloop.rst M Lib/asyncio/base_events.py diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index cf7b6d88725e..5b3d29dcaa46 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -173,10 +173,6 @@ Which clock is used depends on the (platform-specific) event loop implementation; ideally it is a monotonic clock. This will generally be a different clock than :func:`time.time`. -.. note:: - - Timeouts (relative *delay* or absolute *when*) should not exceed one day. - .. method:: AbstractEventLoop.call_later(delay, callback, *args, context=None) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 75989a7641b8..78fe2a719f92 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -63,6 +63,9 @@ _HAS_IPv6 = hasattr(socket, 'AF_INET6') +# Maximum timeout passed to select to avoid OS limitations +MAXIMUM_SELECT_TIMEOUT = 24 * 3600 + def _format_handle(handle): cb = handle._callback @@ -1708,7 +1711,7 @@ def _run_once(self): elif self._scheduled: # Compute the desired timeout. when = self._scheduled[0]._when - timeout = max(0, when - self.time()) + timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT) if self._debug and timeout != 0: t0 = self.time() diff --git a/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst b/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst new file mode 100644 index 000000000000..799463b59163 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-28-17-00-36.bpo-34263.zUfRsu.rst @@ -0,0 +1,2 @@ +asyncio's event loop will not pass timeouts longer than one day to +epoll/select etc.

1 0

[3.6] bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931) (GH-8584)
by Serhiy Storchaka 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/ada5d99306dc8af21c32cefb3d86891e85… commit: ada5d99306dc8af21c32cefb3d86891e8553dbc6 branch: 3.6 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-07-31T12:58:58+03:00 summary: [3.6] bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931) (GH-8584) * Fix integer overflow in os.readv(), os.writev() and in os.sendfile() with headers or trailers arguments (on BSD-based OSes and MacOS). * Fix sending the part of the file in os.sendfile() on MacOS. Using the trailers argument could cause sending more bytes from the input file than was specified. Thanks Ned Deily for testing on 32-bit MacOS. (cherry picked from commit 9d5727326af53ddd91016d98e16ae7cf829caa95) files: A Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst A Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst M Lib/test/test_os.py M Lib/test/test_posix.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 240b7c432ba9..8339f849e84e 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2540,12 +2540,14 @@ class Handler(asynchat.async_chat): def __init__(self, conn): asynchat.async_chat.__init__(self, conn) self.in_buffer = [] + self.accumulate = True self.closed = False self.push(b"220 ready\r\n") def handle_read(self): data = self.recv(4096) - self.in_buffer.append(data) + if self.accumulate: + self.in_buffer.append(data) def get_data(self): return b''.join(self.in_buffer) @@ -2627,6 +2629,8 @@ class TestSendfile(unittest.TestCase): not sys.platform.startswith("sunos") requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') + requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') @classmethod def setUpClass(cls): @@ -2657,17 +2661,13 @@ def tearDown(self): self.server.stop() self.server = None - def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): + def sendfile_wrapper(self, *args, **kwargs): """A higher level wrapper representing how an application is supposed to use sendfile(). """ - while 1: + while True: try: - if self.SUPPORT_HEADERS_TRAILERS: - return os.sendfile(sock, file, offset, nbytes, headers, - trailers) - else: - return os.sendfile(sock, file, offset, nbytes) + return os.sendfile(*args, **kwargs) except OSError as err: if err.errno == errno.ECONNRESET: # disconnected @@ -2758,20 +2758,22 @@ def test_keywords(self): @requires_headers_trailers def test_headers(self): total_sent = 0 + expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1] sent = os.sendfile(self.sockno, self.fileno, 0, 4096, - headers=[b"x" * 512]) + headers=[b"x" * 512, b"y" * 256]) + self.assertLessEqual(sent, 512 + 256 + 4096) total_sent += sent offset = 4096 - nbytes = 4096 - while 1: + while total_sent < len(expected_data): + nbytes = min(len(expected_data) - total_sent, 4096) sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) if sent == 0: break + self.assertLessEqual(sent, nbytes) total_sent += sent offset += sent - expected_data = b"x" * 512 + self.DATA self.assertEqual(total_sent, len(expected_data)) self.client.close() self.server.wait() @@ -2787,12 +2789,30 @@ def test_trailers(self): create_file(TESTFN2, file_data) with open(TESTFN2, 'rb') as f: - os.sendfile(self.sockno, f.fileno(), 0, len(file_data), - trailers=[b"1234"]) + os.sendfile(self.sockno, f.fileno(), 0, 5, + trailers=[b"123456", b"789"]) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() - self.assertEqual(data, b"abcdef1234") + self.assertEqual(data, b"abcde123456789") + + @requires_headers_trailers + @requires_32b + def test_headers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + headers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + @requires_headers_trailers + @requires_32b + def test_trailers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + trailers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) @requires_headers_trailers @unittest.skipUnless(hasattr(os, 'SF_NODISKIO'), diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index eb3ef8494788..a204377d9af4 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -19,6 +19,9 @@ _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(), support.TESTFN + '-dummy-symlink') +requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') + class PosixTester(unittest.TestCase): def setUp(self): @@ -322,6 +325,17 @@ def test_writev(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()") + @requires_32b + def test_writev_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + with self.assertRaises(OSError) as cm: + os.writev(fd, [b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") def test_readv(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -344,6 +358,19 @@ def test_readv(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") + @requires_32b + def test_readv_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + buf = [bytearray(2**16)] * 2**15 + with self.assertRaises(OSError) as cm: + os.readv(fd, buf) + self.assertEqual(cm.exception.errno, errno.EINVAL) + self.assertEqual(bytes(buf[0]), b'\0'* 2**16) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'dup'), 'test needs posix.dup()') def test_dup(self): diff --git a/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst new file mode 100644 index 000000000000..26210fa9b92c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst @@ -0,0 +1,3 @@ +Fixed integer overflow in :func:`os.readv` and :func:`os.writev` +and in :func:`os.sendfile` with *headers* or *trailers* arguments +(on BSD-based OSes and macOS). diff --git a/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst new file mode 100644 index 000000000000..547342c2e9dd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst @@ -0,0 +1,3 @@ +Fixed sending the part of the file in :func:`os.sendfile` on macOS. Using +the *trailers* argument could cause sending more bytes from the input file +than was specified. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 9c05acb24f0c..af6b9dc4ef03 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7924,11 +7924,10 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) -static Py_ssize_t +static int iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { Py_ssize_t i, j; - Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); if (*iov == NULL) { @@ -7953,11 +7952,9 @@ iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, in } Py_DECREF(item); (*iov)[i].iov_base = (*buf)[i].buf; - blen = (*buf)[i].len; - (*iov)[i].iov_len = blen; - total += blen; + (*iov)[i].iov_len = (*buf)[i].len; } - return total; + return 0; fail: PyMem_Del(*iov); @@ -8166,12 +8163,20 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) } if (i > 0) { sf.hdr_cnt = (int)i; - i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE); - if (i < 0) + if (iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE) < 0) return NULL; #ifdef __APPLE__ - sbytes += i; + for (i = 0; i < sf.hdr_cnt; i++) { + Py_ssize_t blen = sf.headers[i].iov_len; +# define OFF_T_MAX 0x7fffffffffffffff + if (sbytes >= OFF_T_MAX - blen) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); + return NULL; + } + sbytes += blen; + } #endif } } @@ -8192,13 +8197,9 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) } if (i > 0) { sf.trl_cnt = (int)i; - i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE); - if (i < 0) + if (iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE) < 0) return NULL; -#ifdef __APPLE__ - sbytes += i; -#endif } } }

1 0

bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931)
by Miss Islington (bot) 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/3e4b68875917a4605b45918f9e3232730f… commit: 3e4b68875917a4605b45918f9e3232730fed9399 branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-07-31T02:20:06-07:00 summary: bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931) * Fix integer overflow in os.readv(), os.writev(), os.preadv() and os.pwritev() and in os.sendfile() with headers or trailers arguments (on BSD-based OSes and MacOS). * Fix sending the part of the file in os.sendfile() on MacOS. Using the trailers argument could cause sending more bytes from the input file than was specified. Thanks Ned Deily for testing on 32-bit MacOS. (cherry picked from commit 9d5727326af53ddd91016d98e16ae7cf829caa95) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst A Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst M Lib/test/test_os.py M Lib/test/test_posix.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e509188243f6..a140ae0d29ea 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2532,12 +2532,14 @@ class Handler(asynchat.async_chat): def __init__(self, conn): asynchat.async_chat.__init__(self, conn) self.in_buffer = [] + self.accumulate = True self.closed = False self.push(b"220 ready\r\n") def handle_read(self): data = self.recv(4096) - self.in_buffer.append(data) + if self.accumulate: + self.in_buffer.append(data) def get_data(self): return b''.join(self.in_buffer) @@ -2618,6 +2620,8 @@ class TestSendfile(unittest.TestCase): not sys.platform.startswith("sunos") requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') + requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') @classmethod def setUpClass(cls): @@ -2648,17 +2652,13 @@ def tearDown(self): self.server.stop() self.server = None - def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): + def sendfile_wrapper(self, *args, **kwargs): """A higher level wrapper representing how an application is supposed to use sendfile(). """ - while 1: + while True: try: - if self.SUPPORT_HEADERS_TRAILERS: - return os.sendfile(sock, file, offset, nbytes, headers, - trailers) - else: - return os.sendfile(sock, file, offset, nbytes) + return os.sendfile(*args, **kwargs) except OSError as err: if err.errno == errno.ECONNRESET: # disconnected @@ -2749,20 +2749,22 @@ def test_keywords(self): @requires_headers_trailers def test_headers(self): total_sent = 0 + expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1] sent = os.sendfile(self.sockno, self.fileno, 0, 4096, - headers=[b"x" * 512]) + headers=[b"x" * 512, b"y" * 256]) + self.assertLessEqual(sent, 512 + 256 + 4096) total_sent += sent offset = 4096 - nbytes = 4096 - while 1: + while total_sent < len(expected_data): + nbytes = min(len(expected_data) - total_sent, 4096) sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) if sent == 0: break + self.assertLessEqual(sent, nbytes) total_sent += sent offset += sent - expected_data = b"x" * 512 + self.DATA self.assertEqual(total_sent, len(expected_data)) self.client.close() self.server.wait() @@ -2778,12 +2780,30 @@ def test_trailers(self): create_file(TESTFN2, file_data) with open(TESTFN2, 'rb') as f: - os.sendfile(self.sockno, f.fileno(), 0, len(file_data), - trailers=[b"1234"]) + os.sendfile(self.sockno, f.fileno(), 0, 5, + trailers=[b"123456", b"789"]) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() - self.assertEqual(data, b"abcdef1234") + self.assertEqual(data, b"abcde123456789") + + @requires_headers_trailers + @requires_32b + def test_headers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + headers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + @requires_headers_trailers + @requires_32b + def test_trailers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + trailers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) @requires_headers_trailers @unittest.skipUnless(hasattr(os, 'SF_NODISKIO'), diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 7dea1beab2c2..d2cb1c9840ea 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -20,6 +20,9 @@ _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(), support.TESTFN + '-dummy-symlink') +requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') + class PosixTester(unittest.TestCase): def setUp(self): @@ -284,6 +287,7 @@ def test_preadv(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'preadv'), "test needs posix.preadv()") @unittest.skipUnless(hasattr(posix, 'RWF_HIPRI'), "test needs posix.RWF_HIPRI") def test_preadv_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -295,6 +299,19 @@ def test_preadv_flags(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'preadv'), "test needs posix.preadv()") + @requires_32b + def test_preadv_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + buf = [bytearray(2**16)] * 2**15 + with self.assertRaises(OSError) as cm: + os.preadv(fd, buf, 0) + self.assertEqual(cm.exception.errno, errno.EINVAL) + self.assertEqual(bytes(buf[0]), b'\0'* 2**16) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()") def test_pwrite(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -320,6 +337,7 @@ def test_pwritev(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwritev'), "test needs posix.pwritev()") @unittest.skipUnless(hasattr(posix, 'os.RWF_SYNC'), "test needs os.RWF_SYNC") def test_pwritev_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -334,6 +352,17 @@ def test_pwritev_flags(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwritev'), "test needs posix.pwritev()") + @requires_32b + def test_pwritev_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + with self.assertRaises(OSError) as cm: + os.pwritev(fd, [b"x" * 2**16] * 2**15, 0) + self.assertEqual(cm.exception.errno, errno.EINVAL) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'posix_fallocate'), "test needs posix.posix_fallocate()") def test_posix_fallocate(self): @@ -435,6 +464,17 @@ def test_writev(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()") + @requires_32b + def test_writev_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + with self.assertRaises(OSError) as cm: + os.writev(fd, [b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") def test_readv(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -457,6 +497,19 @@ def test_readv(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") + @requires_32b + def test_readv_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + buf = [bytearray(2**16)] * 2**15 + with self.assertRaises(OSError) as cm: + os.readv(fd, buf) + self.assertEqual(cm.exception.errno, errno.EINVAL) + self.assertEqual(bytes(buf[0]), b'\0'* 2**16) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'dup'), 'test needs posix.dup()') def test_dup(self): diff --git a/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst new file mode 100644 index 000000000000..9fd15356c72d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst @@ -0,0 +1,3 @@ +Fixed integer overflow in :func:`os.readv`, :func:`os.writev`, +:func:`os.preadv` and :func:`os.pwritev` and in :func:`os.sendfile` with +*headers* or *trailers* arguments (on BSD-based OSes and macOS). diff --git a/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst new file mode 100644 index 000000000000..547342c2e9dd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst @@ -0,0 +1,3 @@ +Fixed sending the part of the file in :func:`os.sendfile` on macOS. Using +the *trailers* argument could cause sending more bytes from the input file +than was specified. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c3682b4a0fd5..d487e5386e89 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8026,12 +8026,13 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) } #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ - || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) -static Py_ssize_t + || defined(__APPLE__))) \ + || defined(HAVE_READV) || defined(HAVE_PREADV) || defined (HAVE_PREADV2) \ + || defined(HAVE_WRITEV) || defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2) +static int iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { Py_ssize_t i, j; - Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); if (*iov == NULL) { @@ -8056,11 +8057,9 @@ iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, in } Py_DECREF(item); (*iov)[i].iov_base = (*buf)[i].buf; - blen = (*buf)[i].len; - (*iov)[i].iov_len = blen; - total += blen; + (*iov)[i].iov_len = (*buf)[i].len; } - return total; + return 0; fail: PyMem_Del(*iov); @@ -8357,12 +8356,20 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) } if (i > 0) { sf.hdr_cnt = (int)i; - i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE); - if (i < 0) + if (iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE) < 0) return NULL; #ifdef __APPLE__ - sbytes += i; + for (i = 0; i < sf.hdr_cnt; i++) { + Py_ssize_t blen = sf.headers[i].iov_len; +# define OFF_T_MAX 0x7fffffffffffffff + if (sbytes >= OFF_T_MAX - blen) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); + return NULL; + } + sbytes += blen; + } #endif } } @@ -8383,13 +8390,9 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) } if (i > 0) { sf.trl_cnt = (int)i; - i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE); - if (i < 0) + if (iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE) < 0) return NULL; -#ifdef __APPLE__ - sbytes += i; -#endif } } }

1 0

Daily reference leaks (4243df51fe43): sum=8
by solipsis＠pitrou.net 31 Jul '18

31 Jul '18

results for 4243df51fe43 on branch "default" -------------------------------------------- test_collections leaked [7, -7, 1] memory blocks, sum=1 test_functools leaked [0, 3, 1] memory blocks, sum=4 test_multiprocessing_fork leaked [2, -1, 0] memory blocks, sum=1 test_multiprocessing_forkserver leaked [2, 0, 0] memory blocks, sum=2 Command line was: ['./python', '-m', 'test.regrtest', '-uall', '-R', '3:3:/home/psf-users/antoine/refleaks/reflogoR3ThN', '--timeout', '7200']

1 0

bpo-33089: Add math.dist() for computing the Euclidean distance between two points (GH-8561)
by Raymond Hettinger 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/9c18b1ae527346bc178250ad1ca07bffda… commit: 9c18b1ae527346bc178250ad1ca07bffdacde5dd branch: master author: Raymond Hettinger <rhettinger(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2018-07-31T00:45:49-07:00 summary: bpo-33089: Add math.dist() for computing the Euclidean distance between two points (GH-8561) files: A Misc/NEWS.d/next/Library/2018-07-29-21-53-15.bpo-33089.hxbp3g.rst M Doc/library/math.rst M Lib/test/test_math.py M Modules/clinic/mathmodule.c.h M Modules/mathmodule.c diff --git a/Doc/library/math.rst b/Doc/library/math.rst index e60d02919c8f..76226c282d5e 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -330,6 +330,18 @@ Trigonometric functions Return the cosine of *x* radians. +.. function:: dist(p, q) + + Return the Euclidean distance between two points *p* and *q*, each + given as a tuple of coordinates. The two tuples must be the same size. + + Roughly equivalent to:: + + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) + + .. versionadded:: 3.8 + + .. function:: hypot(*coordinates) Return the Euclidean norm, ``sqrt(sum(x**2 for x in coordinates))``. diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 484389921a69..448110b1674a 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -4,9 +4,11 @@ from test.support import run_unittest, verbose, requires_IEEE_754 from test import support import unittest +import itertools import math import os import platform +import random import struct import sys import sysconfig @@ -787,6 +789,107 @@ def testHypot(self): scale = FLOAT_MIN / 2.0 ** exp self.assertEqual(math.hypot(4*scale, 3*scale), 5*scale) + def testDist(self): + from decimal import Decimal as D + from fractions import Fraction as F + + dist = math.dist + sqrt = math.sqrt + + # Simple exact case + self.assertEqual(dist((1, 2, 3), (4, 2, -1)), 5.0) + + # Test different numbers of arguments (from zero to nine) + # against a straightforward pure python implementation + for i in range(9): + for j in range(5): + p = tuple(random.uniform(-5, 5) for k in range(i)) + q = tuple(random.uniform(-5, 5) for k in range(i)) + self.assertAlmostEqual( + dist(p, q), + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) + ) + + # Test allowable types (those with __float__) + self.assertEqual(dist((14.0, 1.0), (2.0, -4.0)), 13.0) + self.assertEqual(dist((14, 1), (2, -4)), 13) + self.assertEqual(dist((D(14), D(1)), (D(2), D(-4))), D(13)) + self.assertEqual(dist((F(14, 32), F(1, 32)), (F(2, 32), F(-4, 32))), + F(13, 32)) + self.assertEqual(dist((True, True, False, True, False), + (True, False, True, True, False)), + sqrt(2.0)) + + # Test corner cases + self.assertEqual(dist((13.25, 12.5, -3.25), + (13.25, 12.5, -3.25)), + 0.0) # Distance with self is zero + self.assertEqual(dist((), ()), 0.0) # Zero-dimensional case + self.assertEqual(1.0, # Convert negative zero to positive zero + math.copysign(1.0, dist((-0.0,), (0.0,))) + ) + self.assertEqual(1.0, # Convert negative zero to positive zero + math.copysign(1.0, dist((0.0,), (-0.0,))) + ) + + # Verify tuple subclasses are allowed + class T(tuple): # tuple subclas + pass + self.assertEqual(dist(T((1, 2, 3)), ((4, 2, -1))), 5.0) + + # Test handling of bad arguments + with self.assertRaises(TypeError): # Reject keyword args + dist(p=(1, 2, 3), q=(4, 5, 6)) + with self.assertRaises(TypeError): # Too few args + dist((1, 2, 3)) + with self.assertRaises(TypeError): # Too many args + dist((1, 2, 3), (4, 5, 6), (7, 8, 9)) + with self.assertRaises(TypeError): # Scalars not allowed + dist(1, 2) + with self.assertRaises(TypeError): # Lists not allowed + dist([1, 2, 3], [4, 5, 6]) + with self.assertRaises(TypeError): # Reject values without __float__ + dist((1.1, 'string', 2.2), (1, 2, 3)) + with self.assertRaises(ValueError): # Check dimension agree + dist((1, 2, 3, 4), (5, 6, 7)) + with self.assertRaises(ValueError): # Check dimension agree + dist((1, 2, 3), (4, 5, 6, 7)) + + + # Verify that the one dimensional case equivalent to abs() + for i in range(20): + p, q = random.random(), random.random() + self.assertEqual(dist((p,), (q,)), abs(p - q)) + + # Test special values + values = [NINF, -10.5, -0.0, 0.0, 10.5, INF, NAN] + for p in itertools.product(values, repeat=3): + for q in itertools.product(values, repeat=3): + diffs = [px - qx for px, qx in zip(p, q)] + if any(map(math.isinf, diffs)): + # Any infinite difference gives positive infinity. + self.assertEqual(dist(p, q), INF) + elif any(map(math.isnan, diffs)): + # If no infinity, any NaN gives a Nan. + self.assertTrue(math.isnan(dist(p, q))) + + # Verify scaling for extremely large values + fourthmax = FLOAT_MAX / 4.0 + for n in range(32): + p = (fourthmax,) * n + q = (0.0,) * n + self.assertEqual(dist(p, q), fourthmax * math.sqrt(n)) + self.assertEqual(dist(q, p), fourthmax * math.sqrt(n)) + + # Verify scaling for extremely small values + for exp in range(32): + scale = FLOAT_MIN / 2.0 ** exp + p = (4*scale, 3*scale) + q = (0.0, 0.0) + self.assertEqual(math.dist(p, q), 5*scale) + self.assertEqual(math.dist(q, p), 5*scale) + + def testLdexp(self): self.assertRaises(TypeError, math.ldexp) self.ftest('ldexp(0,1)', math.ldexp(0,1), 0) diff --git a/Misc/NEWS.d/next/Library/2018-07-29-21-53-15.bpo-33089.hxbp3g.rst b/Misc/NEWS.d/next/Library/2018-07-29-21-53-15.bpo-33089.hxbp3g.rst new file mode 100644 index 000000000000..9b253d0bc7f4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-29-21-53-15.bpo-33089.hxbp3g.rst @@ -0,0 +1 @@ +Add math.dist() to compute the Euclidean distance between two points. diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 3a487bdbea23..c4d278634101 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -269,6 +269,41 @@ math_fmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +PyDoc_STRVAR(math_dist__doc__, +"dist($module, p, q, /)\n" +"--\n" +"\n" +"Return the Euclidean distance between two points p and q.\n" +"\n" +"The points should be specified as tuples of coordinates.\n" +"Both tuples must be the same size.\n" +"\n" +"Roughly equivalent to:\n" +" sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q)))"); + +#define MATH_DIST_METHODDEF \ + {"dist", (PyCFunction)math_dist, METH_FASTCALL, math_dist__doc__}, + +static PyObject * +math_dist_impl(PyObject *module, PyObject *p, PyObject *q); + +static PyObject * +math_dist(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *p; + PyObject *q; + + if (!_PyArg_ParseStack(args, nargs, "O!O!:dist", + &PyTuple_Type, &p, &PyTuple_Type, &q)) { + goto exit; + } + return_value = math_dist_impl(module, p, q); + +exit: + return return_value; +} + PyDoc_STRVAR(math_pow__doc__, "pow($module, x, y, /)\n" "--\n" @@ -487,4 +522,4 @@ math_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject exit: return return_value; } -/*[clinic end generated code: output=1c35516a10443902 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d936137c1189b89b input=a9049054013a1b77]*/ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 726fc562129e..8f11f2cf3184 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2031,6 +2031,89 @@ math_fmod_impl(PyObject *module, double x, double y) return PyFloat_FromDouble(r); } +/*[clinic input] +math.dist + + p: object(subclass_of='&PyTuple_Type') + q: object(subclass_of='&PyTuple_Type') + / + +Return the Euclidean distance between two points p and q. + +The points should be specified as tuples of coordinates. +Both tuples must be the same size. + +Roughly equivalent to: + sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) +[clinic start generated code]*/ + +static PyObject * +math_dist_impl(PyObject *module, PyObject *p, PyObject *q) +/*[clinic end generated code: output=56bd9538d06bbcfe input=937122eaa5f19272]*/ +{ + PyObject *item; + double *diffs; + double max = 0.0; + double csum = 0.0; + double x, px, qx, result; + Py_ssize_t i, m, n; + int found_nan = 0; + + m = PyTuple_GET_SIZE(p); + n = PyTuple_GET_SIZE(q); + if (m != n) { + PyErr_SetString(PyExc_ValueError, + "both points must have the same number of dimensions"); + return NULL; + + } + diffs = (double *) PyObject_Malloc(n * sizeof(double)); + if (diffs == NULL) { + return NULL; + } + for (i=0 ; i<n ; i++) { + item = PyTuple_GET_ITEM(p, i); + px = PyFloat_AsDouble(item); + if (px == -1.0 && PyErr_Occurred()) { + PyObject_Free(diffs); + return NULL; + } + item = PyTuple_GET_ITEM(q, i); + qx = PyFloat_AsDouble(item); + if (qx == -1.0 && PyErr_Occurred()) { + PyObject_Free(diffs); + return NULL; + } + x = fabs(px - qx); + diffs[i] = x; + found_nan |= Py_IS_NAN(x); + if (x > max) { + max = x; + } + } + if (Py_IS_INFINITY(max)) { + result = max; + goto done; + } + if (found_nan) { + result = Py_NAN; + goto done; + } + if (max == 0.0) { + result = 0.0; + goto done; + } + for (i=0 ; i<n ; i++) { + x = diffs[i] / max; + csum += x * x; + } + result = max * sqrt(csum); + + done: + PyObject_Free(diffs); + return PyFloat_FromDouble(result); +} + /* AC: cannot convert yet, waiting for *args support */ static PyObject * math_hypot(PyObject *self, PyObject *args) @@ -2358,6 +2441,7 @@ static PyMethodDef math_methods[] = { {"cos", math_cos, METH_O, math_cos_doc}, {"cosh", math_cosh, METH_O, math_cosh_doc}, MATH_DEGREES_METHODDEF + MATH_DIST_METHODDEF {"erf", math_erf, METH_O, math_erf_doc}, {"erfc", math_erfc, METH_O, math_erfc_doc}, {"exp", math_exp, METH_O, math_exp_doc},

1 0

bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931)
by Serhiy Storchaka 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/9d5727326af53ddd91016d98e16ae7cf82… commit: 9d5727326af53ddd91016d98e16ae7cf829caa95 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-07-31T10:24:54+03:00 summary: bpo-33871: Fix os.sendfile(), os.writev(), os.readv(), etc. (GH-7931) * Fix integer overflow in os.readv(), os.writev(), os.preadv() and os.pwritev() and in os.sendfile() with headers or trailers arguments (on BSD-based OSes and MacOS). * Fix sending the part of the file in os.sendfile() on MacOS. Using the trailers argument could cause sending more bytes from the input file than was specified. Thanks Ned Deily for testing on 32-bit MacOS. files: A Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst A Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst M Lib/test/test_os.py M Lib/test/test_posix.py M Modules/posixmodule.c diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e509188243f6..a140ae0d29ea 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2532,12 +2532,14 @@ class Handler(asynchat.async_chat): def __init__(self, conn): asynchat.async_chat.__init__(self, conn) self.in_buffer = [] + self.accumulate = True self.closed = False self.push(b"220 ready\r\n") def handle_read(self): data = self.recv(4096) - self.in_buffer.append(data) + if self.accumulate: + self.in_buffer.append(data) def get_data(self): return b''.join(self.in_buffer) @@ -2618,6 +2620,8 @@ class TestSendfile(unittest.TestCase): not sys.platform.startswith("sunos") requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') + requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') @classmethod def setUpClass(cls): @@ -2648,17 +2652,13 @@ def tearDown(self): self.server.stop() self.server = None - def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): + def sendfile_wrapper(self, *args, **kwargs): """A higher level wrapper representing how an application is supposed to use sendfile(). """ - while 1: + while True: try: - if self.SUPPORT_HEADERS_TRAILERS: - return os.sendfile(sock, file, offset, nbytes, headers, - trailers) - else: - return os.sendfile(sock, file, offset, nbytes) + return os.sendfile(*args, **kwargs) except OSError as err: if err.errno == errno.ECONNRESET: # disconnected @@ -2749,20 +2749,22 @@ def test_keywords(self): @requires_headers_trailers def test_headers(self): total_sent = 0 + expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1] sent = os.sendfile(self.sockno, self.fileno, 0, 4096, - headers=[b"x" * 512]) + headers=[b"x" * 512, b"y" * 256]) + self.assertLessEqual(sent, 512 + 256 + 4096) total_sent += sent offset = 4096 - nbytes = 4096 - while 1: + while total_sent < len(expected_data): + nbytes = min(len(expected_data) - total_sent, 4096) sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) if sent == 0: break + self.assertLessEqual(sent, nbytes) total_sent += sent offset += sent - expected_data = b"x" * 512 + self.DATA self.assertEqual(total_sent, len(expected_data)) self.client.close() self.server.wait() @@ -2778,12 +2780,30 @@ def test_trailers(self): create_file(TESTFN2, file_data) with open(TESTFN2, 'rb') as f: - os.sendfile(self.sockno, f.fileno(), 0, len(file_data), - trailers=[b"1234"]) + os.sendfile(self.sockno, f.fileno(), 0, 5, + trailers=[b"123456", b"789"]) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() - self.assertEqual(data, b"abcdef1234") + self.assertEqual(data, b"abcde123456789") + + @requires_headers_trailers + @requires_32b + def test_headers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + headers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + @requires_headers_trailers + @requires_32b + def test_trailers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + trailers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) @requires_headers_trailers @unittest.skipUnless(hasattr(os, 'SF_NODISKIO'), diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 4f6abc6f7d6f..e569f7ae72da 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -20,6 +20,9 @@ _DUMMY_SYMLINK = os.path.join(tempfile.gettempdir(), support.TESTFN + '-dummy-symlink') +requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') + class PosixTester(unittest.TestCase): def setUp(self): @@ -284,6 +287,7 @@ def test_preadv(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'preadv'), "test needs posix.preadv()") @unittest.skipUnless(hasattr(posix, 'RWF_HIPRI'), "test needs posix.RWF_HIPRI") def test_preadv_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -295,6 +299,19 @@ def test_preadv_flags(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'preadv'), "test needs posix.preadv()") + @requires_32b + def test_preadv_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + buf = [bytearray(2**16)] * 2**15 + with self.assertRaises(OSError) as cm: + os.preadv(fd, buf, 0) + self.assertEqual(cm.exception.errno, errno.EINVAL) + self.assertEqual(bytes(buf[0]), b'\0'* 2**16) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()") def test_pwrite(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -320,6 +337,7 @@ def test_pwritev(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwritev'), "test needs posix.pwritev()") @unittest.skipUnless(hasattr(posix, 'os.RWF_SYNC'), "test needs os.RWF_SYNC") def test_pwritev_flags(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -334,6 +352,17 @@ def test_pwritev_flags(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'pwritev'), "test needs posix.pwritev()") + @requires_32b + def test_pwritev_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + with self.assertRaises(OSError) as cm: + os.pwritev(fd, [b"x" * 2**16] * 2**15, 0) + self.assertEqual(cm.exception.errno, errno.EINVAL) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'posix_fallocate'), "test needs posix.posix_fallocate()") def test_posix_fallocate(self): @@ -435,6 +464,17 @@ def test_writev(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()") + @requires_32b + def test_writev_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + with self.assertRaises(OSError) as cm: + os.writev(fd, [b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") def test_readv(self): fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) @@ -457,6 +497,19 @@ def test_readv(self): finally: os.close(fd) + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") + @requires_32b + def test_readv_overflow_32bits(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + buf = [bytearray(2**16)] * 2**15 + with self.assertRaises(OSError) as cm: + os.readv(fd, buf) + self.assertEqual(cm.exception.errno, errno.EINVAL) + self.assertEqual(bytes(buf[0]), b'\0'* 2**16) + finally: + os.close(fd) + @unittest.skipUnless(hasattr(posix, 'dup'), 'test needs posix.dup()') def test_dup(self): diff --git a/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst new file mode 100644 index 000000000000..9fd15356c72d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-06-26-19-03-56.bpo-33871.XhlrGU.rst @@ -0,0 +1,3 @@ +Fixed integer overflow in :func:`os.readv`, :func:`os.writev`, +:func:`os.preadv` and :func:`os.pwritev` and in :func:`os.sendfile` with +*headers* or *trailers* arguments (on BSD-based OSes and macOS). diff --git a/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst new file mode 100644 index 000000000000..547342c2e9dd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2018-06-26-19-35-33.bpo-33871.S4HR9n.rst @@ -0,0 +1,3 @@ +Fixed sending the part of the file in :func:`os.sendfile` on macOS. Using +the *trailers* argument could cause sending more bytes from the input file +than was specified. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 435e5f61271c..6104c0f74dee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8321,12 +8321,13 @@ os_read_impl(PyObject *module, int fd, Py_ssize_t length) } #if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ - || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) -static Py_ssize_t + || defined(__APPLE__))) \ + || defined(HAVE_READV) || defined(HAVE_PREADV) || defined (HAVE_PREADV2) \ + || defined(HAVE_WRITEV) || defined(HAVE_PWRITEV) || defined (HAVE_PWRITEV2) +static int iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, int type) { Py_ssize_t i, j; - Py_ssize_t blen, total = 0; *iov = PyMem_New(struct iovec, cnt); if (*iov == NULL) { @@ -8351,11 +8352,9 @@ iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, Py_ssize_t cnt, in } Py_DECREF(item); (*iov)[i].iov_base = (*buf)[i].buf; - blen = (*buf)[i].len; - (*iov)[i].iov_len = blen; - total += blen; + (*iov)[i].iov_len = (*buf)[i].len; } - return total; + return 0; fail: PyMem_Del(*iov); @@ -8652,12 +8651,20 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) } if (i > 0) { sf.hdr_cnt = (int)i; - i = iov_setup(&(sf.headers), &hbuf, - headers, sf.hdr_cnt, PyBUF_SIMPLE); - if (i < 0) + if (iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE) < 0) return NULL; #ifdef __APPLE__ - sbytes += i; + for (i = 0; i < sf.hdr_cnt; i++) { + Py_ssize_t blen = sf.headers[i].iov_len; +# define OFF_T_MAX 0x7fffffffffffffff + if (sbytes >= OFF_T_MAX - blen) { + PyErr_SetString(PyExc_OverflowError, + "sendfile() header is too large"); + return NULL; + } + sbytes += blen; + } #endif } } @@ -8678,13 +8685,9 @@ posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) } if (i > 0) { sf.trl_cnt = (int)i; - i = iov_setup(&(sf.trailers), &tbuf, - trailers, sf.trl_cnt, PyBUF_SIMPLE); - if (i < 0) + if (iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE) < 0) return NULL; -#ifdef __APPLE__ - sbytes += i; -#endif } } }

1 0

[3.7] bpo-33729: Fix issues with arguments parsing in hashlib. (GH-8346) (GH-8581)
by Serhiy Storchaka 31 Jul '18

31 Jul '18

https://github.com/python/cpython/commit/47957dab94a4efa2fee61c9a8193f78300… commit: 47957dab94a4efa2fee61c9a8193f78300950769 branch: 3.7 author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2018-07-31T10:22:44+03:00 summary: [3.7] bpo-33729: Fix issues with arguments parsing in hashlib. (GH-8346) (GH-8581) * help(hashlib) didn't work because of incorrect module name in blake2b and blake2s classes. * Constructors blake2*(), sha3_*(), shake_*() and keccak_*() incorrectly accepted keyword argument "string" for binary data, but documented as accepting the "data" keyword argument. Now this parameter is positional-only. * Keyword-only parameters in blake2b() and blake2s() were not documented as keyword-only. * Default value for some parameters of blake2b() and blake2s() was None, which is not acceptable value. * The length argument for shake_*.digest() was wrapped out to 32 bits. * The argument for shake_128.digest() and shake_128.hexdigest() was not positional-only as intended. * TypeError messages for incorrect arguments in all constructors sha3_*(), shake_*() and keccak_*() incorrectly referred to sha3_224. Also made the following enhancements: * More accurately specified input and result types for strings, bytes and bytes-like objects. * Unified positional parameter names for update() and constructors. * Improved formatting. (cherry picked from commit f1d36d8efaecd5c84cb35e35119b283f37d83c40) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2018-07-20-09-11-05.bpo-33729.sO6iTb.rst M Doc/library/hashlib.rst M Lib/hashlib.py M Lib/test/test_hashlib.py M Modules/_blake2/blake2b_impl.c M Modules/_blake2/blake2s_impl.c M Modules/_blake2/clinic/blake2b_impl.c.h M Modules/_blake2/clinic/blake2s_impl.c.h M Modules/_hashopenssl.c M Modules/_sha3/clinic/sha3module.c.h M Modules/_sha3/sha3module.c M Modules/clinic/_hashopenssl.c.h diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index da941e5bb42b..baf6b0af242c 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -101,7 +101,7 @@ More condensed: .. function:: new(name[, data]) - Is a generic constructor that takes the string name of the desired + Is a generic constructor that takes the string *name* of the desired algorithm as its first parameter. It also exists to allow access to the above listed hashes as well as any other algorithms that your OpenSSL library may offer. The named constructors are much faster than :func:`new` @@ -162,10 +162,10 @@ A hash object has the following attributes: A hash object has the following methods: -.. method:: hash.update(arg) +.. method:: hash.update(data) - Update the hash object with the object *arg*, which must be interpretable as - a buffer of bytes. Repeated calls are equivalent to a single call with the + Update the hash object with the :term:`bytes-like object`. + Repeated calls are equivalent to a single call with the concatenation of all the arguments: ``m.update(a); m.update(b)`` is equivalent to ``m.update(a+b)``. @@ -206,7 +206,7 @@ by the SHAKE algorithm. .. method:: shake.digest(length) Return the digest of the data passed to the :meth:`update` method so far. - This is a bytes object of size ``length`` which may contain bytes in + This is a bytes object of size *length* which may contain bytes in the whole range from 0 to 255. @@ -262,9 +262,10 @@ include a `salt <https://en.wikipedia.org/wiki/Salt_%28cryptography%29>`_. The function provides scrypt password-based key derivation function as defined in :rfc:`7914`. - *password* and *salt* must be bytes-like objects. Applications and - libraries should limit *password* to a sensible length (e.g. 1024). *salt* - should be about 16 or more bytes from a proper source, e.g. :func:`os.urandom`. + *password* and *salt* must be :term:`bytes-like objects + <bytes-like object>`. Applications and libraries should limit *password* + to a sensible length (e.g. 1024). *salt* should be about 16 or more + bytes from a proper source, e.g. :func:`os.urandom`. *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). @@ -305,11 +306,11 @@ Creating hash objects New hash objects are created by calling constructor functions: -.. function:: blake2b(data=b'', digest_size=64, key=b'', salt=b'', \ +.. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \ person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ node_depth=0, inner_size=0, last_node=False) -.. function:: blake2s(data=b'', digest_size=32, key=b'', salt=b'', \ +.. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \ person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ node_depth=0, inner_size=0, last_node=False) @@ -317,8 +318,8 @@ New hash objects are created by calling constructor functions: These functions return the corresponding hash objects for calculating BLAKE2b or BLAKE2s. They optionally take these general parameters: -* *data*: initial chunk of data to hash, which must be interpretable as buffer - of bytes. +* *data*: initial chunk of data to hash, which must be + :term:`bytes-like object`. It can be passed only as positional argument. * *digest_size*: size of output digest in bytes. @@ -427,7 +428,7 @@ object, and, finally, get the digest out of the object by calling As a shortcut, you can pass the first chunk of data to update directly to the -constructor as the first argument (or as *data* keyword argument): +constructor as the positional argument: >>> from hashlib import blake2b >>> blake2b(b'Hello world').hexdigest() diff --git a/Lib/hashlib.py b/Lib/hashlib.py index e30c6100d7db..4e783a86a34e 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -25,18 +25,18 @@ sha384 and sha512 will be slow on 32 bit platforms. Hash objects have these methods: - - update(arg): Update the hash object with the bytes in arg. Repeated calls - are equivalent to a single call with the concatenation of all - the arguments. - - digest(): Return the digest of the bytes passed to the update() method - so far. - - hexdigest(): Like digest() except the digest is returned as a unicode - object of double length, containing only hexadecimal digits. - - copy(): Return a copy (clone) of the hash object. This can be used to - efficiently compute the digests of strings that share a common - initial substring. - -For example, to obtain the digest of the string 'Nobody inspects the + - update(data): Update the hash object with the bytes in data. Repeated calls + are equivalent to a single call with the concatenation of all + the arguments. + - digest(): Return the digest of the bytes passed to the update() method + so far as a bytes object. + - hexdigest(): Like digest() except the digest is returned as a string + of double length, containing only hexadecimal digits. + - copy(): Return a copy (clone) of the hash object. This can be used to + efficiently compute the digests of datas that share a common + initial substring. + +For example, to obtain the digest of the byte string 'Nobody inspects the spammish repetition': >>> import hashlib @@ -130,14 +130,15 @@ def __get_openssl_constructor(name): def __py_new(name, data=b'', **kwargs): """new(name, data=b'', **kwargs) - Return a new hashing object using the - named algorithm; optionally initialized with data (which must be bytes). + named algorithm; optionally initialized with data (which must be + a bytes-like object). """ return __get_builtin_constructor(name)(data, **kwargs) def __hash_new(name, data=b'', **kwargs): """new(name, data=b'') - Return a new hashing object using the named algorithm; - optionally initialized with data (which must be bytes). + optionally initialized with data (which must be a bytes-like object). """ if name in {'blake2b', 'blake2s'}: # Prefer our blake2 implementation. diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5b0218b08567..5b7f4082ae04 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -568,8 +568,12 @@ def check_blake2(self, constructor, salt_size, person_size, key_size, self.assertRaises(OverflowError, constructor, node_offset=-1) self.assertRaises(OverflowError, constructor, node_offset=max_offset+1) + self.assertRaises(TypeError, constructor, data=b'') + self.assertRaises(TypeError, constructor, string=b'') + self.assertRaises(TypeError, constructor, '') + constructor( - string=b'', + b'', key=b'', salt=b'', person=b'', diff --git a/Misc/NEWS.d/next/Library/2018-07-20-09-11-05.bpo-33729.sO6iTb.rst b/Misc/NEWS.d/next/Library/2018-07-20-09-11-05.bpo-33729.sO6iTb.rst new file mode 100644 index 000000000000..c4718722ad7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-20-09-11-05.bpo-33729.sO6iTb.rst @@ -0,0 +1 @@ +Fixed issues with arguments parsing in :mod:`hashlib`. diff --git a/Modules/_blake2/blake2b_impl.c b/Modules/_blake2/blake2b_impl.c index 822b9f285caf..f143b5d0c2ff 100644 --- a/Modules/_blake2/blake2b_impl.c +++ b/Modules/_blake2/blake2b_impl.c @@ -47,10 +47,10 @@ typedef struct { #include "clinic/blake2b_impl.c.h" /*[clinic input] -module _blake2b -class _blake2b.blake2b "BLAKE2bObject *" "&PyBlake2_BLAKE2bType" +module _blake2 +class _blake2.blake2b "BLAKE2bObject *" "&PyBlake2_BLAKE2bType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=6893358c6622aecf]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=d47b0527b39c673f]*/ static BLAKE2bObject * @@ -66,17 +66,18 @@ new_BLAKE2bObject(PyTypeObject *type) /*[clinic input] @classmethod -_blake2b.blake2b.__new__ as py_blake2b_new - string as data: object = NULL +_blake2.blake2b.__new__ as py_blake2b_new + data: object(c_default="NULL") = b'' + / * - digest_size: int(c_default="BLAKE2B_OUTBYTES") = _blake2b.blake2b.MAX_DIGEST_SIZE - key: Py_buffer = None - salt: Py_buffer = None - person: Py_buffer = None + digest_size: int(c_default="BLAKE2B_OUTBYTES") = _blake2.blake2b.MAX_DIGEST_SIZE + key: Py_buffer(c_default="NULL", py_default="b''") = None + salt: Py_buffer(c_default="NULL", py_default="b''") = None + person: Py_buffer(c_default="NULL", py_default="b''") = None fanout: int = 1 depth: int = 1 - leaf_size as leaf_size_obj: object = NULL - node_offset as node_offset_obj: object = NULL + leaf_size as leaf_size_obj: object(c_default="NULL") = 0 + node_offset as node_offset_obj: object(c_default="NULL") = 0 node_depth: int = 0 inner_size: int = 0 last_node: bool = False @@ -90,7 +91,7 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, int fanout, int depth, PyObject *leaf_size_obj, PyObject *node_offset_obj, int node_depth, int inner_size, int last_node) -/*[clinic end generated code: output=7506d8d890e5f13b input=e41548dfa0866031]*/ +/*[clinic end generated code: output=7506d8d890e5f13b input=aca35b33c5612b4b]*/ { BLAKE2bObject *self = NULL; Py_buffer buf; @@ -252,14 +253,14 @@ py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, } /*[clinic input] -_blake2b.blake2b.copy +_blake2.blake2b.copy Return a copy of the hash object. [clinic start generated code]*/ static PyObject * -_blake2b_blake2b_copy_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=c89cd33550ab1543 input=4c9c319f18f10747]*/ +_blake2_blake2b_copy_impl(BLAKE2bObject *self) +/*[clinic end generated code: output=ff6acee5f93656ae input=e383c2d199fd8a2e]*/ { BLAKE2bObject *cpy; @@ -274,21 +275,21 @@ _blake2b_blake2b_copy_impl(BLAKE2bObject *self) } /*[clinic input] -_blake2b.blake2b.update +_blake2.blake2b.update - obj: object + data: object / -Update this hash object's state with the provided string. +Update this hash object's state with the provided bytes-like object. [clinic start generated code]*/ static PyObject * -_blake2b_blake2b_update(BLAKE2bObject *self, PyObject *obj) -/*[clinic end generated code: output=a888f07c4cddbe94 input=3ecb8c13ee4260f2]*/ +_blake2_blake2b_update(BLAKE2bObject *self, PyObject *data) +/*[clinic end generated code: output=010dfcbe22654359 input=ffc4aa6a6a225d31]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) self->lock = PyThread_allocate_lock(); @@ -308,14 +309,14 @@ _blake2b_blake2b_update(BLAKE2bObject *self, PyObject *obj) } /*[clinic input] -_blake2b.blake2b.digest +_blake2.blake2b.digest -Return the digest value as a string of binary data. +Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * -_blake2b_blake2b_digest_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=b13a79360d984740 input=ac2fa462ebb1b9c7]*/ +_blake2_blake2b_digest_impl(BLAKE2bObject *self) +/*[clinic end generated code: output=a5864660f4bfc61a input=7d21659e9c5fff02]*/ { uint8_t digest[BLAKE2B_OUTBYTES]; blake2b_state state_cpy; @@ -329,14 +330,14 @@ _blake2b_blake2b_digest_impl(BLAKE2bObject *self) } /*[clinic input] -_blake2b.blake2b.hexdigest +_blake2.blake2b.hexdigest Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * -_blake2b_blake2b_hexdigest_impl(BLAKE2bObject *self) -/*[clinic end generated code: output=6a503611715b24bd input=d58f0b2f37812e33]*/ +_blake2_blake2b_hexdigest_impl(BLAKE2bObject *self) +/*[clinic end generated code: output=b5598a87d8794a60 input=76930f6946351f56]*/ { uint8_t digest[BLAKE2B_OUTBYTES]; blake2b_state state_cpy; @@ -350,10 +351,10 @@ _blake2b_blake2b_hexdigest_impl(BLAKE2bObject *self) static PyMethodDef py_blake2b_methods[] = { - _BLAKE2B_BLAKE2B_COPY_METHODDEF - _BLAKE2B_BLAKE2B_DIGEST_METHODDEF - _BLAKE2B_BLAKE2B_HEXDIGEST_METHODDEF - _BLAKE2B_BLAKE2B_UPDATE_METHODDEF + _BLAKE2_BLAKE2B_COPY_METHODDEF + _BLAKE2_BLAKE2B_DIGEST_METHODDEF + _BLAKE2_BLAKE2B_HEXDIGEST_METHODDEF + _BLAKE2_BLAKE2B_UPDATE_METHODDEF {NULL, NULL} }; diff --git a/Modules/_blake2/blake2s_impl.c b/Modules/_blake2/blake2s_impl.c index 3b76c9cd89d7..7d366d394005 100644 --- a/Modules/_blake2/blake2s_impl.c +++ b/Modules/_blake2/blake2s_impl.c @@ -47,10 +47,10 @@ typedef struct { #include "clinic/blake2s_impl.c.h" /*[clinic input] -module _blake2s -class _blake2s.blake2s "BLAKE2sObject *" "&PyBlake2_BLAKE2sType" +module _blake2 +class _blake2.blake2s "BLAKE2sObject *" "&PyBlake2_BLAKE2sType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=edbfcf7557a685a7]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b79d7ffe07286ce]*/ static BLAKE2sObject * @@ -66,17 +66,18 @@ new_BLAKE2sObject(PyTypeObject *type) /*[clinic input] @classmethod -_blake2s.blake2s.__new__ as py_blake2s_new - string as data: object = NULL +_blake2.blake2s.__new__ as py_blake2s_new + data: object(c_default="NULL") = b'' + / * - digest_size: int(c_default="BLAKE2S_OUTBYTES") = _blake2s.blake2s.MAX_DIGEST_SIZE - key: Py_buffer = None - salt: Py_buffer = None - person: Py_buffer = None + digest_size: int(c_default="BLAKE2S_OUTBYTES") = _blake2.blake2s.MAX_DIGEST_SIZE + key: Py_buffer(c_default="NULL", py_default="b''") = None + salt: Py_buffer(c_default="NULL", py_default="b''") = None + person: Py_buffer(c_default="NULL", py_default="b''") = None fanout: int = 1 depth: int = 1 - leaf_size as leaf_size_obj: object = NULL - node_offset as node_offset_obj: object = NULL + leaf_size as leaf_size_obj: object(c_default="NULL") = 0 + node_offset as node_offset_obj: object(c_default="NULL") = 0 node_depth: int = 0 inner_size: int = 0 last_node: bool = False @@ -90,7 +91,7 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, int fanout, int depth, PyObject *leaf_size_obj, PyObject *node_offset_obj, int node_depth, int inner_size, int last_node) -/*[clinic end generated code: output=fe060b258a8cbfc6 input=458cfdcb3d0d47ff]*/ +/*[clinic end generated code: output=fe060b258a8cbfc6 input=3abfaabe7f5f62cc]*/ { BLAKE2sObject *self = NULL; Py_buffer buf; @@ -252,14 +253,14 @@ py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, } /*[clinic input] -_blake2s.blake2s.copy +_blake2.blake2s.copy Return a copy of the hash object. [clinic start generated code]*/ static PyObject * -_blake2s_blake2s_copy_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=6c5bada404b7aed7 input=c8858e887ae4a07a]*/ +_blake2_blake2s_copy_impl(BLAKE2sObject *self) +/*[clinic end generated code: output=5b90131c4eae275e input=0b9d44942f0fe4b2]*/ { BLAKE2sObject *cpy; @@ -274,21 +275,21 @@ _blake2s_blake2s_copy_impl(BLAKE2sObject *self) } /*[clinic input] -_blake2s.blake2s.update +_blake2.blake2s.update - obj: object + data: object / -Update this hash object's state with the provided string. +Update this hash object's state with the provided bytes-like object. [clinic start generated code]*/ static PyObject * -_blake2s_blake2s_update(BLAKE2sObject *self, PyObject *obj) -/*[clinic end generated code: output=fe8438a1d3cede87 input=47a408b9a3cc05c5]*/ +_blake2_blake2s_update(BLAKE2sObject *self, PyObject *data) +/*[clinic end generated code: output=757dc087fec37815 input=97500db2f9de4aaa]*/ { Py_buffer buf; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) self->lock = PyThread_allocate_lock(); @@ -308,14 +309,14 @@ _blake2s_blake2s_update(BLAKE2sObject *self, PyObject *obj) } /*[clinic input] -_blake2s.blake2s.digest +_blake2.blake2s.digest -Return the digest value as a string of binary data. +Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * -_blake2s_blake2s_digest_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=80e81a48c6f79cf9 input=feb9a220135bdeba]*/ +_blake2_blake2s_digest_impl(BLAKE2sObject *self) +/*[clinic end generated code: output=40c566ca4bc6bc51 input=f41e0b8d6d937454]*/ { uint8_t digest[BLAKE2S_OUTBYTES]; blake2s_state state_cpy; @@ -329,14 +330,14 @@ _blake2s_blake2s_digest_impl(BLAKE2sObject *self) } /*[clinic input] -_blake2s.blake2s.hexdigest +_blake2.blake2s.hexdigest Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * -_blake2s_blake2s_hexdigest_impl(BLAKE2sObject *self) -/*[clinic end generated code: output=db6c5028c0a3c2e5 input=4e4877b8bd7aea91]*/ +_blake2_blake2s_hexdigest_impl(BLAKE2sObject *self) +/*[clinic end generated code: output=15153eb5e59c52eb input=c77a1321567e8952]*/ { uint8_t digest[BLAKE2S_OUTBYTES]; blake2s_state state_cpy; @@ -350,10 +351,10 @@ _blake2s_blake2s_hexdigest_impl(BLAKE2sObject *self) static PyMethodDef py_blake2s_methods[] = { - _BLAKE2S_BLAKE2S_COPY_METHODDEF - _BLAKE2S_BLAKE2S_DIGEST_METHODDEF - _BLAKE2S_BLAKE2S_HEXDIGEST_METHODDEF - _BLAKE2S_BLAKE2S_UPDATE_METHODDEF + _BLAKE2_BLAKE2S_COPY_METHODDEF + _BLAKE2_BLAKE2S_DIGEST_METHODDEF + _BLAKE2_BLAKE2S_HEXDIGEST_METHODDEF + _BLAKE2_BLAKE2S_UPDATE_METHODDEF {NULL, NULL} }; diff --git a/Modules/_blake2/clinic/blake2b_impl.c.h b/Modules/_blake2/clinic/blake2b_impl.c.h index 71c073aa36c6..9b2965eb6b31 100644 --- a/Modules/_blake2/clinic/blake2b_impl.c.h +++ b/Modules/_blake2/clinic/blake2b_impl.c.h @@ -3,10 +3,9 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(py_blake2b_new__doc__, -"blake2b(string=None, *, digest_size=_blake2b.blake2b.MAX_DIGEST_SIZE,\n" -" key=None, salt=None, person=None, fanout=1, depth=1,\n" -" leaf_size=None, node_offset=None, node_depth=0, inner_size=0,\n" -" last_node=False)\n" +"blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" +" key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" +" node_offset=0, node_depth=0, inner_size=0, last_node=False)\n" "--\n" "\n" "Return a new BLAKE2b hash object."); @@ -22,7 +21,7 @@ static PyObject * py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"string", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; + static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiip:blake2b", _keywords, 0}; PyObject *data = NULL; int digest_size = BLAKE2B_OUTBYTES; @@ -60,66 +59,66 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } -PyDoc_STRVAR(_blake2b_blake2b_copy__doc__, +PyDoc_STRVAR(_blake2_blake2b_copy__doc__, "copy($self, /)\n" "--\n" "\n" "Return a copy of the hash object."); -#define _BLAKE2B_BLAKE2B_COPY_METHODDEF \ - {"copy", (PyCFunction)_blake2b_blake2b_copy, METH_NOARGS, _blake2b_blake2b_copy__doc__}, +#define _BLAKE2_BLAKE2B_COPY_METHODDEF \ + {"copy", (PyCFunction)_blake2_blake2b_copy, METH_NOARGS, _blake2_blake2b_copy__doc__}, static PyObject * -_blake2b_blake2b_copy_impl(BLAKE2bObject *self); +_blake2_blake2b_copy_impl(BLAKE2bObject *self); static PyObject * -_blake2b_blake2b_copy(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_copy(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2b_blake2b_copy_impl(self); + return _blake2_blake2b_copy_impl(self); } -PyDoc_STRVAR(_blake2b_blake2b_update__doc__, -"update($self, obj, /)\n" +PyDoc_STRVAR(_blake2_blake2b_update__doc__, +"update($self, data, /)\n" "--\n" "\n" -"Update this hash object\'s state with the provided string."); +"Update this hash object\'s state with the provided bytes-like object."); -#define _BLAKE2B_BLAKE2B_UPDATE_METHODDEF \ - {"update", (PyCFunction)_blake2b_blake2b_update, METH_O, _blake2b_blake2b_update__doc__}, +#define _BLAKE2_BLAKE2B_UPDATE_METHODDEF \ + {"update", (PyCFunction)_blake2_blake2b_update, METH_O, _blake2_blake2b_update__doc__}, -PyDoc_STRVAR(_blake2b_blake2b_digest__doc__, +PyDoc_STRVAR(_blake2_blake2b_digest__doc__, "digest($self, /)\n" "--\n" "\n" -"Return the digest value as a string of binary data."); +"Return the digest value as a bytes object."); -#define _BLAKE2B_BLAKE2B_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_blake2b_blake2b_digest, METH_NOARGS, _blake2b_blake2b_digest__doc__}, +#define _BLAKE2_BLAKE2B_DIGEST_METHODDEF \ + {"digest", (PyCFunction)_blake2_blake2b_digest, METH_NOARGS, _blake2_blake2b_digest__doc__}, static PyObject * -_blake2b_blake2b_digest_impl(BLAKE2bObject *self); +_blake2_blake2b_digest_impl(BLAKE2bObject *self); static PyObject * -_blake2b_blake2b_digest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_digest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2b_blake2b_digest_impl(self); + return _blake2_blake2b_digest_impl(self); } -PyDoc_STRVAR(_blake2b_blake2b_hexdigest__doc__, +PyDoc_STRVAR(_blake2_blake2b_hexdigest__doc__, "hexdigest($self, /)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); -#define _BLAKE2B_BLAKE2B_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_blake2b_blake2b_hexdigest, METH_NOARGS, _blake2b_blake2b_hexdigest__doc__}, +#define _BLAKE2_BLAKE2B_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)_blake2_blake2b_hexdigest, METH_NOARGS, _blake2_blake2b_hexdigest__doc__}, static PyObject * -_blake2b_blake2b_hexdigest_impl(BLAKE2bObject *self); +_blake2_blake2b_hexdigest_impl(BLAKE2bObject *self); static PyObject * -_blake2b_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2b_hexdigest(BLAKE2bObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2b_blake2b_hexdigest_impl(self); + return _blake2_blake2b_hexdigest_impl(self); } -/*[clinic end generated code: output=535a54852c98e51c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0eb559f418fc0a21 input=a9049054013a1b77]*/ diff --git a/Modules/_blake2/clinic/blake2s_impl.c.h b/Modules/_blake2/clinic/blake2s_impl.c.h index ca908d393b5b..42b87b7099df 100644 --- a/Modules/_blake2/clinic/blake2s_impl.c.h +++ b/Modules/_blake2/clinic/blake2s_impl.c.h @@ -3,10 +3,9 @@ preserve [clinic start generated code]*/ PyDoc_STRVAR(py_blake2s_new__doc__, -"blake2s(string=None, *, digest_size=_blake2s.blake2s.MAX_DIGEST_SIZE,\n" -" key=None, salt=None, person=None, fanout=1, depth=1,\n" -" leaf_size=None, node_offset=None, node_depth=0, inner_size=0,\n" -" last_node=False)\n" +"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" +" key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" +" node_offset=0, node_depth=0, inner_size=0, last_node=False)\n" "--\n" "\n" "Return a new BLAKE2s hash object."); @@ -22,7 +21,7 @@ static PyObject * py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"string", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; + static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", NULL}; static _PyArg_Parser _parser = {"|O$iy*y*y*iiOOiip:blake2s", _keywords, 0}; PyObject *data = NULL; int digest_size = BLAKE2S_OUTBYTES; @@ -60,66 +59,66 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } -PyDoc_STRVAR(_blake2s_blake2s_copy__doc__, +PyDoc_STRVAR(_blake2_blake2s_copy__doc__, "copy($self, /)\n" "--\n" "\n" "Return a copy of the hash object."); -#define _BLAKE2S_BLAKE2S_COPY_METHODDEF \ - {"copy", (PyCFunction)_blake2s_blake2s_copy, METH_NOARGS, _blake2s_blake2s_copy__doc__}, +#define _BLAKE2_BLAKE2S_COPY_METHODDEF \ + {"copy", (PyCFunction)_blake2_blake2s_copy, METH_NOARGS, _blake2_blake2s_copy__doc__}, static PyObject * -_blake2s_blake2s_copy_impl(BLAKE2sObject *self); +_blake2_blake2s_copy_impl(BLAKE2sObject *self); static PyObject * -_blake2s_blake2s_copy(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2s_copy(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2s_blake2s_copy_impl(self); + return _blake2_blake2s_copy_impl(self); } -PyDoc_STRVAR(_blake2s_blake2s_update__doc__, -"update($self, obj, /)\n" +PyDoc_STRVAR(_blake2_blake2s_update__doc__, +"update($self, data, /)\n" "--\n" "\n" -"Update this hash object\'s state with the provided string."); +"Update this hash object\'s state with the provided bytes-like object."); -#define _BLAKE2S_BLAKE2S_UPDATE_METHODDEF \ - {"update", (PyCFunction)_blake2s_blake2s_update, METH_O, _blake2s_blake2s_update__doc__}, +#define _BLAKE2_BLAKE2S_UPDATE_METHODDEF \ + {"update", (PyCFunction)_blake2_blake2s_update, METH_O, _blake2_blake2s_update__doc__}, -PyDoc_STRVAR(_blake2s_blake2s_digest__doc__, +PyDoc_STRVAR(_blake2_blake2s_digest__doc__, "digest($self, /)\n" "--\n" "\n" -"Return the digest value as a string of binary data."); +"Return the digest value as a bytes object."); -#define _BLAKE2S_BLAKE2S_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_blake2s_blake2s_digest, METH_NOARGS, _blake2s_blake2s_digest__doc__}, +#define _BLAKE2_BLAKE2S_DIGEST_METHODDEF \ + {"digest", (PyCFunction)_blake2_blake2s_digest, METH_NOARGS, _blake2_blake2s_digest__doc__}, static PyObject * -_blake2s_blake2s_digest_impl(BLAKE2sObject *self); +_blake2_blake2s_digest_impl(BLAKE2sObject *self); static PyObject * -_blake2s_blake2s_digest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2s_digest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2s_blake2s_digest_impl(self); + return _blake2_blake2s_digest_impl(self); } -PyDoc_STRVAR(_blake2s_blake2s_hexdigest__doc__, +PyDoc_STRVAR(_blake2_blake2s_hexdigest__doc__, "hexdigest($self, /)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); -#define _BLAKE2S_BLAKE2S_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_blake2s_blake2s_hexdigest, METH_NOARGS, _blake2s_blake2s_hexdigest__doc__}, +#define _BLAKE2_BLAKE2S_HEXDIGEST_METHODDEF \ + {"hexdigest", (PyCFunction)_blake2_blake2s_hexdigest, METH_NOARGS, _blake2_blake2s_hexdigest__doc__}, static PyObject * -_blake2s_blake2s_hexdigest_impl(BLAKE2sObject *self); +_blake2_blake2s_hexdigest_impl(BLAKE2sObject *self); static PyObject * -_blake2s_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) +_blake2_blake2s_hexdigest(BLAKE2sObject *self, PyObject *Py_UNUSED(ignored)) { - return _blake2s_blake2s_hexdigest_impl(self); + return _blake2_blake2s_hexdigest_impl(self); } -/*[clinic end generated code: output=535ea7903f9ccf76 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=13d4b08ea9ee2d62 input=a9049054013a1b77]*/ diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index b6dcc0699233..40cd6327312d 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -729,6 +729,10 @@ pbkdf2_hmac(PyObject *self, PyObject *args, PyObject *kwdict) #if OPENSSL_VERSION_NUMBER > 0x10100000L && !defined(OPENSSL_NO_SCRYPT) && !defined(LIBRESSL_VERSION_NUMBER) #define PY_SCRYPT 1 +/* XXX: Parameters salt, n, r and p should be required keyword-only parameters. + They are optional in the Argument Clinic declaration only due to a + limitation of PyArg_ParseTupleAndKeywords. */ + /*[clinic input] _hashlib.scrypt @@ -858,13 +862,13 @@ _hashlib.hmac_digest msg: Py_buffer digest: str -Single-shot HMAC +Single-shot HMAC. [clinic start generated code]*/ static PyObject * _hashlib_hmac_digest_impl(PyObject *module, Py_buffer *key, Py_buffer *msg, const char *digest) -/*[clinic end generated code: output=75630e684cdd8762 input=10e964917921e2f2]*/ +/*[clinic end generated code: output=75630e684cdd8762 input=562d2f4249511bd3]*/ { unsigned char md[EVP_MAX_MD_SIZE] = {0}; unsigned int md_len = 0; diff --git a/Modules/_sha3/clinic/sha3module.c.h b/Modules/_sha3/clinic/sha3module.c.h index a1e80b061e58..35f11abca0fc 100644 --- a/Modules/_sha3/clinic/sha3module.c.h +++ b/Modules/_sha3/clinic/sha3module.c.h @@ -2,33 +2,6 @@ preserve [clinic start generated code]*/ -PyDoc_STRVAR(py_sha3_new__doc__, -"sha3_224(string=None)\n" -"--\n" -"\n" -"Return a new SHA3 hash object with a hashbit length of 28 bytes."); - -static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data); - -static PyObject * -py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - static const char * const _keywords[] = {"string", NULL}; - static _PyArg_Parser _parser = {"|O:sha3_224", _keywords, 0}; - PyObject *data = NULL; - - if (!_PyArg_ParseTupleAndKeywordsFast(args, kwargs, &_parser, - &data)) { - goto exit; - } - return_value = py_sha3_new_impl(type, data); - -exit: - return return_value; -} - PyDoc_STRVAR(_sha3_sha3_224_copy__doc__, "copy($self, /)\n" "--\n" @@ -51,7 +24,7 @@ PyDoc_STRVAR(_sha3_sha3_224_digest__doc__, "digest($self, /)\n" "--\n" "\n" -"Return the digest value as a string of binary data."); +"Return the digest value as a bytes object."); #define _SHA3_SHA3_224_DIGEST_METHODDEF \ {"digest", (PyCFunction)_sha3_sha3_224_digest, METH_NOARGS, _sha3_sha3_224_digest__doc__}, @@ -84,71 +57,29 @@ _sha3_sha3_224_hexdigest(SHA3object *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(_sha3_sha3_224_update__doc__, -"update($self, obj, /)\n" +"update($self, data, /)\n" "--\n" "\n" -"Update this hash object\'s state with the provided string."); +"Update this hash object\'s state with the provided bytes-like object."); #define _SHA3_SHA3_224_UPDATE_METHODDEF \ {"update", (PyCFunction)_sha3_sha3_224_update, METH_O, _sha3_sha3_224_update__doc__}, PyDoc_STRVAR(_sha3_shake_128_digest__doc__, -"digest($self, /, length)\n" +"digest($self, length, /)\n" "--\n" "\n" -"Return the digest value as a string of binary data."); +"Return the digest value as a bytes object."); #define _SHA3_SHAKE_128_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_sha3_shake_128_digest, METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_digest__doc__}, - -static PyObject * -_sha3_shake_128_digest_impl(SHA3object *self, unsigned long length); - -static PyObject * -_sha3_shake_128_digest(SHA3object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - static const char * const _keywords[] = {"length", NULL}; - static _PyArg_Parser _parser = {"k:digest", _keywords, 0}; - unsigned long length; - - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &length)) { - goto exit; - } - return_value = _sha3_shake_128_digest_impl(self, length); - -exit: - return return_value; -} + {"digest", (PyCFunction)_sha3_shake_128_digest, METH_O, _sha3_shake_128_digest__doc__}, PyDoc_STRVAR(_sha3_shake_128_hexdigest__doc__, -"hexdigest($self, /, length)\n" +"hexdigest($self, length, /)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); #define _SHA3_SHAKE_128_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_sha3_shake_128_hexdigest, METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_hexdigest__doc__}, - -static PyObject * -_sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length); - -static PyObject * -_sha3_shake_128_hexdigest(SHA3object *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) -{ - PyObject *return_value = NULL; - static const char * const _keywords[] = {"length", NULL}; - static _PyArg_Parser _parser = {"k:hexdigest", _keywords, 0}; - unsigned long length; - - if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &length)) { - goto exit; - } - return_value = _sha3_shake_128_hexdigest_impl(self, length); - -exit: - return return_value; -} -/*[clinic end generated code: output=a3aeb6c3b2fbd905 input=a9049054013a1b77]*/ + {"hexdigest", (PyCFunction)_sha3_shake_128_hexdigest, METH_O, _sha3_shake_128_hexdigest__doc__}, +/*[clinic end generated code: output=826b6b5a7c3406eb input=a9049054013a1b77]*/ diff --git a/Modules/_sha3/sha3module.c b/Modules/_sha3/sha3module.c index 970ce1616e57..f73bc99eafe9 100644 --- a/Modules/_sha3/sha3module.c +++ b/Modules/_sha3/sha3module.c @@ -170,21 +170,20 @@ newSHA3object(PyTypeObject *type) } -/*[clinic input] -@classmethod -_sha3.sha3_224.__new__ as py_sha3_new - string as data: object = NULL - -Return a new SHA3 hash object with a hashbit length of 28 bytes. -[clinic start generated code]*/ - static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data) -/*[clinic end generated code: output=8d5c34279e69bf09 input=d7c582b950a858b6]*/ +py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { SHA3object *self = NULL; Py_buffer buf = {NULL, NULL}; HashReturn res; + PyObject *data = NULL; + + if (!_PyArg_NoKeywords(_PyType_Name(type), kwargs)) { + return NULL; + } + if (!PyArg_UnpackTuple(args, _PyType_Name(type), 0, 1, &data)) { + return NULL; + } self = newSHA3object(type); if (self == NULL) { @@ -292,12 +291,12 @@ _sha3_sha3_224_copy_impl(SHA3object *self) /*[clinic input] _sha3.sha3_224.digest -Return the digest value as a string of binary data. +Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * _sha3_sha3_224_digest_impl(SHA3object *self) -/*[clinic end generated code: output=fd531842e20b2d5b input=a5807917d219b30e]*/ +/*[clinic end generated code: output=fd531842e20b2d5b input=5b2a659536bbd248]*/ { unsigned char digest[SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE]; SHA3_state temp; @@ -347,20 +346,20 @@ _sha3_sha3_224_hexdigest_impl(SHA3object *self) /*[clinic input] _sha3.sha3_224.update - obj: object + data: object / -Update this hash object's state with the provided string. +Update this hash object's state with the provided bytes-like object. [clinic start generated code]*/ static PyObject * -_sha3_sha3_224_update(SHA3object *self, PyObject *obj) -/*[clinic end generated code: output=06721d55b483e0af input=be44bf0d1c279791]*/ +_sha3_sha3_224_update(SHA3object *self, PyObject *data) +/*[clinic end generated code: output=d3223352286ed357 input=a887f54dcc4ae227]*/ { Py_buffer buf; HashReturn res; - GET_BUFFER_VIEW_OR_ERROUT(obj, &buf); + GET_BUFFER_VIEW_OR_ERROUT(data, &buf); /* add new data, the function takes the length in bits not bytes */ if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE) { @@ -529,44 +528,49 @@ static PyGetSetDef SHA3_getseters[] = { py_sha3_new, /* tp_new */ \ } +PyDoc_STRVAR(sha3_224__doc__, +"sha3_224([data]) -> SHA3 object\n\ +\n\ +Return a new SHA3 hash object with a hashbit length of 28 bytes."); + PyDoc_STRVAR(sha3_256__doc__, -"sha3_256([string]) -> SHA3 object\n\ +"sha3_256([data]) -> SHA3 object\n\ \n\ Return a new SHA3 hash object with a hashbit length of 32 bytes."); PyDoc_STRVAR(sha3_384__doc__, -"sha3_384([string]) -> SHA3 object\n\ +"sha3_384([data]) -> SHA3 object\n\ \n\ Return a new SHA3 hash object with a hashbit length of 48 bytes."); PyDoc_STRVAR(sha3_512__doc__, -"sha3_512([string]) -> SHA3 object\n\ +"sha3_512([data]) -> SHA3 object\n\ \n\ Return a new SHA3 hash object with a hashbit length of 64 bytes."); -SHA3_TYPE(SHA3_224type, "_sha3.sha3_224", py_sha3_new__doc__, SHA3_methods); +SHA3_TYPE(SHA3_224type, "_sha3.sha3_224", sha3_224__doc__, SHA3_methods); SHA3_TYPE(SHA3_256type, "_sha3.sha3_256", sha3_256__doc__, SHA3_methods); SHA3_TYPE(SHA3_384type, "_sha3.sha3_384", sha3_384__doc__, SHA3_methods); SHA3_TYPE(SHA3_512type, "_sha3.sha3_512", sha3_512__doc__, SHA3_methods); #ifdef PY_WITH_KECCAK PyDoc_STRVAR(keccak_224__doc__, -"keccak_224([string]) -> Keccak object\n\ +"keccak_224([data]) -> Keccak object\n\ \n\ Return a new Keccak hash object with a hashbit length of 28 bytes."); PyDoc_STRVAR(keccak_256__doc__, -"keccak_256([string]) -> Keccak object\n\ +"keccak_256([data]) -> Keccak object\n\ \n\ Return a new Keccak hash object with a hashbit length of 32 bytes."); PyDoc_STRVAR(keccak_384__doc__, -"keccak_384([string]) -> Keccak object\n\ +"keccak_384([data]) -> Keccak object\n\ \n\ Return a new Keccak hash object with a hashbit length of 48 bytes."); PyDoc_STRVAR(keccak_512__doc__, -"keccak_512([string]) -> Keccak object\n\ +"keccak_512([data]) -> Keccak object\n\ \n\ Return a new Keccak hash object with a hashbit length of 64 bytes."); @@ -578,13 +582,19 @@ SHA3_TYPE(Keccak_512type, "_sha3.keccak_512", keccak_512__doc__, SHA3_methods); static PyObject * -_SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) +_SHAKE_digest(SHA3object *self, PyObject *digestlen_obj, int hex) { + unsigned long digestlen; unsigned char *digest = NULL; SHA3_state temp; int res; PyObject *result = NULL; + digestlen = PyLong_AsUnsignedLong(digestlen_obj); + if (digestlen == (unsigned long) -1 && PyErr_Occurred()) { + return NULL; + } + /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and * SHA3_LANESIZE extra space. */ @@ -624,15 +634,15 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) /*[clinic input] _sha3.shake_128.digest - length: unsigned_long(bitwise=True) - \ + length: object + / -Return the digest value as a string of binary data. +Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * -_sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=2313605e2f87bb8f input=608c8ca80ae9d115]*/ +_sha3_shake_128_digest(SHA3object *self, PyObject *length) +/*[clinic end generated code: output=eaa80b6299142396 input=c579eb109f6227d2]*/ { return _SHAKE_digest(self, length, 0); } @@ -641,15 +651,15 @@ _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) /*[clinic input] _sha3.shake_128.hexdigest - length: unsigned_long(bitwise=True) - \ + length: object + / Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * -_sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=bf8e2f1e490944a8 input=64e56b4760db4573]*/ +_sha3_shake_128_hexdigest(SHA3object *self, PyObject *length) +/*[clinic end generated code: output=4752f90e53c8bf2a input=a82694ea83865f5a]*/ { return _SHAKE_digest(self, length, 1); } @@ -664,12 +674,12 @@ static PyMethodDef SHAKE_methods[] = { }; PyDoc_STRVAR(shake_128__doc__, -"shake_128([string]) -> SHAKE object\n\ +"shake_128([data]) -> SHAKE object\n\ \n\ Return a new SHAKE hash object."); PyDoc_STRVAR(shake_256__doc__, -"shake_256([string]) -> SHAKE object\n\ +"shake_256([data]) -> SHAKE object\n\ \n\ Return a new SHAKE hash object."); diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index cbc8638c9468..070f8290f26f 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -58,7 +58,7 @@ PyDoc_STRVAR(_hashlib_hmac_digest__doc__, "hmac_digest($module, /, key, msg, digest)\n" "--\n" "\n" -"Single-shot HMAC"); +"Single-shot HMAC."); #define _HASHLIB_HMAC_DIGEST_METHODDEF \ {"hmac_digest", (PyCFunction)_hashlib_hmac_digest, METH_FASTCALL|METH_KEYWORDS, _hashlib_hmac_digest__doc__}, @@ -99,4 +99,4 @@ _hashlib_hmac_digest(PyObject *module, PyObject *const *args, Py_ssize_t nargs, #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=b5b90821caf05391 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b129f1a6ec7b8503 input=a9049054013a1b77]*/

1 0