Mailman 3 February 2020 - Python-checkins

closes bpo-39621: Make buf arg to md5_compress be const. (GH-18497)
by Andy Lester 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/597ebed748d0b0c061f8c108bd98270d10… commit: 597ebed748d0b0c061f8c108bd98270d103286c1 branch: master author: Andy Lester <andy(a)petdance.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T20:53:01-08:00 summary: closes bpo-39621: Make buf arg to md5_compress be const. (GH-18497) files: M Modules/md5module.c diff --git a/Modules/md5module.c b/Modules/md5module.c index d783ae5a765fa..ea2bafb9b65e8 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -119,7 +119,7 @@ typedef struct { a = (a + I(b,c,d) + M + t); a = ROLc(a, s) + b; -static void md5_compress(struct md5_state *md5, unsigned char *buf) +static void md5_compress(struct md5_state *md5, const unsigned char *buf) { MD5_INT32 i, W[16], a, b, c, d; @@ -242,7 +242,7 @@ md5_process(struct md5_state *md5, const unsigned char *in, Py_ssize_t inlen) while (inlen > 0) { if (md5->curlen == 0 && inlen >= MD5_BLOCKSIZE) { - md5_compress(md5, (unsigned char *)in); + md5_compress(md5, in); md5->length += MD5_BLOCKSIZE * 8; in += MD5_BLOCKSIZE; inlen -= MD5_BLOCKSIZE;

1 0

bpo-35134: Add Include/cpython/fileutils.h header file (GH-18493)
by Victor Stinner 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/8c3aee65ed3aff3668da330ccd6f9ba7b2… commit: 8c3aee65ed3aff3668da330ccd6f9ba7b2aa4567 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-12T23:55:09+01:00 summary: bpo-35134: Add Include/cpython/fileutils.h header file (GH-18493) Move CPython C API from Include/fileutils.h into a new Include/cpython/fileutils.h header file which is included by Include/fileutils.h. Exclude the following private symbols from the limited C API: * _Py_error_handler * _Py_GetErrorHandler() * _Py_DecodeLocaleEx() * _Py_EncodeLocaleEx() files: A Include/cpython/fileutils.h M Include/fileutils.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters diff --git a/Include/cpython/fileutils.h b/Include/cpython/fileutils.h new file mode 100644 index 0000000000000..e79d03e24f577 --- /dev/null +++ b/Include/cpython/fileutils.h @@ -0,0 +1,165 @@ +#ifndef Py_CPYTHON_FILEUTILS_H +# error "this header file must not be included directly" +#endif + +typedef enum { + _Py_ERROR_UNKNOWN=0, + _Py_ERROR_STRICT, + _Py_ERROR_SURROGATEESCAPE, + _Py_ERROR_REPLACE, + _Py_ERROR_IGNORE, + _Py_ERROR_BACKSLASHREPLACE, + _Py_ERROR_SURROGATEPASS, + _Py_ERROR_XMLCHARREFREPLACE, + _Py_ERROR_OTHER +} _Py_error_handler; + +PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); + +PyAPI_FUNC(int) _Py_DecodeLocaleEx( + const char *arg, + wchar_t **wstr, + size_t *wlen, + const char **reason, + int current_locale, + _Py_error_handler errors); + +PyAPI_FUNC(int) _Py_EncodeLocaleEx( + const wchar_t *text, + char **str, + size_t *error_pos, + const char **reason, + int current_locale, + _Py_error_handler errors); + + +PyAPI_FUNC(PyObject *) _Py_device_encoding(int); + +#if defined(MS_WINDOWS) || defined(__APPLE__) + /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). + On macOS 10.13, read() and write() with more than INT_MAX bytes + fail with EINVAL (bpo-24658). */ +# define _PY_READ_MAX INT_MAX +# define _PY_WRITE_MAX INT_MAX +#else + /* write() should truncate the input to PY_SSIZE_T_MAX bytes, + but it's safer to do it ourself to have a portable behaviour */ +# define _PY_READ_MAX PY_SSIZE_T_MAX +# define _PY_WRITE_MAX PY_SSIZE_T_MAX +#endif + +#ifdef MS_WINDOWS +struct _Py_stat_struct { + unsigned long st_dev; + uint64_t st_ino; + unsigned short st_mode; + int st_nlink; + int st_uid; + int st_gid; + unsigned long st_rdev; + __int64 st_size; + time_t st_atime; + int st_atime_nsec; + time_t st_mtime; + int st_mtime_nsec; + time_t st_ctime; + int st_ctime_nsec; + unsigned long st_file_attributes; + unsigned long st_reparse_tag; +}; +#else +# define _Py_stat_struct stat +#endif + +PyAPI_FUNC(int) _Py_fstat( + int fd, + struct _Py_stat_struct *status); + +PyAPI_FUNC(int) _Py_fstat_noraise( + int fd, + struct _Py_stat_struct *status); + +PyAPI_FUNC(int) _Py_stat( + PyObject *path, + struct stat *status); + +PyAPI_FUNC(int) _Py_open( + const char *pathname, + int flags); + +PyAPI_FUNC(int) _Py_open_noraise( + const char *pathname, + int flags); + +PyAPI_FUNC(FILE *) _Py_wfopen( + const wchar_t *path, + const wchar_t *mode); + +PyAPI_FUNC(FILE*) _Py_fopen( + const char *pathname, + const char *mode); + +PyAPI_FUNC(FILE*) _Py_fopen_obj( + PyObject *path, + const char *mode); + +PyAPI_FUNC(Py_ssize_t) _Py_read( + int fd, + void *buf, + size_t count); + +PyAPI_FUNC(Py_ssize_t) _Py_write( + int fd, + const void *buf, + size_t count); + +PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( + int fd, + const void *buf, + size_t count); + +#ifdef HAVE_READLINK +PyAPI_FUNC(int) _Py_wreadlink( + const wchar_t *path, + wchar_t *buf, + /* Number of characters of 'buf' buffer + including the trailing NUL character */ + size_t buflen); +#endif + +#ifdef HAVE_REALPATH +PyAPI_FUNC(wchar_t*) _Py_wrealpath( + const wchar_t *path, + wchar_t *resolved_path, + /* Number of characters of 'resolved_path' buffer + including the trailing NUL character */ + size_t resolved_path_len); +#endif + +#ifndef MS_WINDOWS +PyAPI_FUNC(int) _Py_isabs(const wchar_t *path); +#endif + +PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p); + +PyAPI_FUNC(wchar_t*) _Py_wgetcwd( + wchar_t *buf, + /* Number of characters of 'buf' buffer + including the trailing NUL character */ + size_t buflen); + +PyAPI_FUNC(int) _Py_get_inheritable(int fd); + +PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, + int *atomic_flag_works); + +PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, + int *atomic_flag_works); + +PyAPI_FUNC(int) _Py_dup(int fd); + +#ifndef MS_WINDOWS +PyAPI_FUNC(int) _Py_get_blocking(int fd); + +PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); +#endif /* !MS_WINDOWS */ diff --git a/Include/fileutils.h b/Include/fileutils.h index a9655bbf9a5f0..12bd071c49c04 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -18,173 +18,12 @@ PyAPI_FUNC(char*) _Py_EncodeLocaleRaw( size_t *error_pos); #endif - -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03080000 -typedef enum { - _Py_ERROR_UNKNOWN=0, - _Py_ERROR_STRICT, - _Py_ERROR_SURROGATEESCAPE, - _Py_ERROR_REPLACE, - _Py_ERROR_IGNORE, - _Py_ERROR_BACKSLASHREPLACE, - _Py_ERROR_SURROGATEPASS, - _Py_ERROR_XMLCHARREFREPLACE, - _Py_ERROR_OTHER -} _Py_error_handler; - -PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); - -PyAPI_FUNC(int) _Py_DecodeLocaleEx( - const char *arg, - wchar_t **wstr, - size_t *wlen, - const char **reason, - int current_locale, - _Py_error_handler errors); - -PyAPI_FUNC(int) _Py_EncodeLocaleEx( - const wchar_t *text, - char **str, - size_t *error_pos, - const char **reason, - int current_locale, - _Py_error_handler errors); -#endif - #ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _Py_device_encoding(int); - -#if defined(MS_WINDOWS) || defined(__APPLE__) - /* On Windows, the count parameter of read() is an int (bpo-9015, bpo-9611). - On macOS 10.13, read() and write() with more than INT_MAX bytes - fail with EINVAL (bpo-24658). */ -# define _PY_READ_MAX INT_MAX -# define _PY_WRITE_MAX INT_MAX -#else - /* write() should truncate the input to PY_SSIZE_T_MAX bytes, - but it's safer to do it ourself to have a portable behaviour */ -# define _PY_READ_MAX PY_SSIZE_T_MAX -# define _PY_WRITE_MAX PY_SSIZE_T_MAX +# define Py_CPYTHON_FILEUTILS_H +# include "cpython/fileutils.h" +# undef Py_CPYTHON_FILEUTILS_H #endif -#ifdef MS_WINDOWS -struct _Py_stat_struct { - unsigned long st_dev; - uint64_t st_ino; - unsigned short st_mode; - int st_nlink; - int st_uid; - int st_gid; - unsigned long st_rdev; - __int64 st_size; - time_t st_atime; - int st_atime_nsec; - time_t st_mtime; - int st_mtime_nsec; - time_t st_ctime; - int st_ctime_nsec; - unsigned long st_file_attributes; - unsigned long st_reparse_tag; -}; -#else -# define _Py_stat_struct stat -#endif - -PyAPI_FUNC(int) _Py_fstat( - int fd, - struct _Py_stat_struct *status); - -PyAPI_FUNC(int) _Py_fstat_noraise( - int fd, - struct _Py_stat_struct *status); - -PyAPI_FUNC(int) _Py_stat( - PyObject *path, - struct stat *status); - -PyAPI_FUNC(int) _Py_open( - const char *pathname, - int flags); - -PyAPI_FUNC(int) _Py_open_noraise( - const char *pathname, - int flags); - -PyAPI_FUNC(FILE *) _Py_wfopen( - const wchar_t *path, - const wchar_t *mode); - -PyAPI_FUNC(FILE*) _Py_fopen( - const char *pathname, - const char *mode); - -PyAPI_FUNC(FILE*) _Py_fopen_obj( - PyObject *path, - const char *mode); - -PyAPI_FUNC(Py_ssize_t) _Py_read( - int fd, - void *buf, - size_t count); - -PyAPI_FUNC(Py_ssize_t) _Py_write( - int fd, - const void *buf, - size_t count); - -PyAPI_FUNC(Py_ssize_t) _Py_write_noraise( - int fd, - const void *buf, - size_t count); - -#ifdef HAVE_READLINK -PyAPI_FUNC(int) _Py_wreadlink( - const wchar_t *path, - wchar_t *buf, - /* Number of characters of 'buf' buffer - including the trailing NUL character */ - size_t buflen); -#endif - -#ifdef HAVE_REALPATH -PyAPI_FUNC(wchar_t*) _Py_wrealpath( - const wchar_t *path, - wchar_t *resolved_path, - /* Number of characters of 'resolved_path' buffer - including the trailing NUL character */ - size_t resolved_path_len); -#endif - -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_isabs(const wchar_t *path); -#endif - -PyAPI_FUNC(int) _Py_abspath(const wchar_t *path, wchar_t **abspath_p); - -PyAPI_FUNC(wchar_t*) _Py_wgetcwd( - wchar_t *buf, - /* Number of characters of 'buf' buffer - including the trailing NUL character */ - size_t buflen); - -PyAPI_FUNC(int) _Py_get_inheritable(int fd); - -PyAPI_FUNC(int) _Py_set_inheritable(int fd, int inheritable, - int *atomic_flag_works); - -PyAPI_FUNC(int) _Py_set_inheritable_async_safe(int fd, int inheritable, - int *atomic_flag_works); - -PyAPI_FUNC(int) _Py_dup(int fd); - -#ifndef MS_WINDOWS -PyAPI_FUNC(int) _Py_get_blocking(int fd); - -PyAPI_FUNC(int) _Py_set_blocking(int fd, int blocking); -#endif /* !MS_WINDOWS */ - -#endif /* Py_LIMITED_API */ - #ifdef __cplusplus } #endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 2f3fab7b33ccb..3199a1aa02d65 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1060,6 +1060,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/ceval.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ + $(srcdir)/Include/cpython/fileutils.h \ $(srcdir)/Include/cpython/import.h \ $(srcdir)/Include/cpython/initconfig.h \ $(srcdir)/Include/cpython/interpreteridobject.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9e6323fdcbb0d..0acf7f4a8de80 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -131,6 +131,7 @@ <ClInclude Include="..\Include\cpython\ceval.h" /> <ClInclude Include="..\Include\cpython\dictobject.h" /> <ClInclude Include="..\Include\cpython\fileobject.h" /> + <ClInclude Include="..\Include\cpython\fileutils.h" /> <ClInclude Include="..\Include\cpython\import.h" /> <ClInclude Include="..\Include\cpython\initconfig.h" /> <ClInclude Include="..\Include\cpython\listobject.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index b1412d550c05a..a846a37630a1c 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -96,6 +96,9 @@ <ClInclude Include="..\Include\cpython\fileobject.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\cpython\fileutils.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\cpython\import.h"> <Filter>Include</Filter> </ClInclude>

1 0

bpo-35134: Add Include/cpython/bytesobject.h file (GH-18494)
by Victor Stinner 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/98921aeaf5879b51e2dd1870c9285cfa8d… commit: 98921aeaf5879b51e2dd1870c9285cfa8d1a52c7 branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-12T23:54:31+01:00 summary: bpo-35134: Add Include/cpython/bytesobject.h file (GH-18494) Add Include/cpython/bytearrayobject.h and Include/cpython/bytesobject.h header files. Move CPython C API from Include/bytesobject.h into a new Include/cpython/bytesobject.h header file which is included by Include/bytesobject.h. Do a similar change for Include/bytearrayobject.h. files: A Include/cpython/bytearrayobject.h A Include/cpython/bytesobject.h M Include/bytearrayobject.h M Include/bytesobject.h M Makefile.pre.in M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 647a17a819d23..341ab38a15d5a 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -18,17 +18,6 @@ extern "C" { * to contain a char pointer, not an unsigned char pointer. */ -/* Object layout */ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_VAR_HEAD - Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ - char *ob_bytes; /* Physical backing buffer */ - char *ob_start; /* Logical start inside ob_bytes */ - Py_ssize_t ob_exports; /* How many buffer exports */ -} PyByteArrayObject; -#endif - /* Type object */ PyAPI_DATA(PyTypeObject) PyByteArray_Type; PyAPI_DATA(PyTypeObject) PyByteArrayIter_Type; @@ -45,14 +34,10 @@ PyAPI_FUNC(Py_ssize_t) PyByteArray_Size(PyObject *); PyAPI_FUNC(char *) PyByteArray_AsString(PyObject *); PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); -/* Macros, trading safety for speed */ #ifndef Py_LIMITED_API -#define PyByteArray_AS_STRING(self) \ - (assert(PyByteArray_Check(self)), \ - Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string) -#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self)) - -PyAPI_DATA(char) _PyByteArray_empty_string[]; +# define Py_CPYTHON_BYTEARRAYOBJECT_H +# include "cpython/bytearrayobject.h" +# undef Py_CPYTHON_BYTEARRAYOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 4aaa71a832bd7..27c31ee342c88 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -27,20 +27,6 @@ functions should be applied to nil objects. /* Caching the hash (ob_shash) saves recalculation of a string's hash value. This significantly speeds up dict lookups. */ -#ifndef Py_LIMITED_API -typedef struct { - PyObject_VAR_HEAD - Py_hash_t ob_shash; - char ob_sval[1]; - - /* Invariants: - * ob_sval contains space for 'ob_size+1' elements. - * ob_sval[ob_size] == 0. - * ob_shash is the hash of the string or -1 if not computed yet. - */ -} PyBytesObject; -#endif - PyAPI_DATA(PyTypeObject) PyBytes_Type; PyAPI_DATA(PyTypeObject) PyBytesIter_Type; @@ -60,38 +46,9 @@ PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *); PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); -#ifndef Py_LIMITED_API -PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( - const char *format, - Py_ssize_t format_len, - PyObject *args, - int use_bytearray); -PyAPI_FUNC(PyObject*) _PyBytes_FromHex( - PyObject *string, - int use_bytearray); -#endif PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, const char *, Py_ssize_t, const char *); -#ifndef Py_LIMITED_API -/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ -PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, const char **); -#endif - -/* Macro, trading safety for speed */ -#ifndef Py_LIMITED_API -#define PyBytes_AS_STRING(op) (assert(PyBytes_Check(op)), \ - (((PyBytesObject *)(op))->ob_sval)) -#define PyBytes_GET_SIZE(op) (assert(PyBytes_Check(op)),Py_SIZE(op)) -#endif - -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -#ifndef Py_LIMITED_API -PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); -#endif /* Provides access to the internal data buffer and size of a string object or the default encoded version of a Unicode object. Passing @@ -114,85 +71,10 @@ PyAPI_FUNC(int) PyBytes_AsStringAndSize( #define F_ZERO (1<<4) #ifndef Py_LIMITED_API -/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". - A _PyBytesWriter variable must be declared at the end of variables in a - function to optimize the memory allocation on the stack. */ -typedef struct { - /* bytes, bytearray or NULL (when the small buffer is used) */ - PyObject *buffer; - - /* Number of allocated size. */ - Py_ssize_t allocated; - - /* Minimum number of allocated bytes, - incremented by _PyBytesWriter_Prepare() */ - Py_ssize_t min_size; - - /* If non-zero, use a bytearray instead of a bytes object for buffer. */ - int use_bytearray; - - /* If non-zero, overallocate the buffer (default: 0). - This flag must be zero if use_bytearray is non-zero. */ - int overallocate; - - /* Stack buffer */ - int use_small_buffer; - char small_buffer[512]; -} _PyBytesWriter; - -/* Initialize a bytes writer - - By default, the overallocation is disabled. Set the overallocate attribute - to control the allocation of the buffer. */ -PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); - -/* Get the buffer content and reset the writer. - Return a bytes object, or a bytearray object if use_bytearray is non-zero. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, - void *str); - -/* Deallocate memory of a writer (clear its internal buffer). */ -PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); - -/* Allocate the buffer to write size bytes. - Return the pointer to the beginning of buffer data. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, - Py_ssize_t size); - -/* Ensure that the buffer is large enough to write *size* bytes. - Add size to the writer minimum size (min_size attribute). - - str is the current pointer inside the buffer. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Resize the buffer to make it larger. - The new buffer may be larger than size bytes because of overallocation. - Return the updated current pointer inside the buffer. - Raise an exception and return NULL on error. - - Note: size must be greater than the number of allocated bytes in the writer. - - This function doesn't use the writer minimum size (min_size attribute). - - See also _PyBytesWriter_Prepare(). - */ -PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, - void *str, - Py_ssize_t size); - -/* Write bytes. - Raise an exception and return NULL on error. */ -PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, - void *str, - const void *bytes, - Py_ssize_t size); -#endif /* Py_LIMITED_API */ +# define Py_CPYTHON_BYTESOBJECT_H +# include "cpython/bytesobject.h" +# undef Py_CPYTHON_BYTESOBJECT_H +#endif #ifdef __cplusplus } diff --git a/Include/cpython/bytearrayobject.h b/Include/cpython/bytearrayobject.h new file mode 100644 index 0000000000000..569b0cd036986 --- /dev/null +++ b/Include/cpython/bytearrayobject.h @@ -0,0 +1,20 @@ +#ifndef Py_CPYTHON_BYTEARRAYOBJECT_H +# error "this header file must not be included directly" +#endif + +/* Object layout */ +typedef struct { + PyObject_VAR_HEAD + Py_ssize_t ob_alloc; /* How many bytes allocated in ob_bytes */ + char *ob_bytes; /* Physical backing buffer */ + char *ob_start; /* Logical start inside ob_bytes */ + Py_ssize_t ob_exports; /* How many buffer exports */ +} PyByteArrayObject; + +/* Macros, trading safety for speed */ +#define PyByteArray_AS_STRING(self) \ + (assert(PyByteArray_Check(self)), \ + Py_SIZE(self) ? ((PyByteArrayObject *)(self))->ob_start : _PyByteArray_empty_string) +#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)), Py_SIZE(self)) + +PyAPI_DATA(char) _PyByteArray_empty_string[]; diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h new file mode 100644 index 0000000000000..f284c5835df09 --- /dev/null +++ b/Include/cpython/bytesobject.h @@ -0,0 +1,118 @@ +#ifndef Py_CPYTHON_BYTESOBJECT_H +# error "this header file must not be included directly" +#endif + +typedef struct { + PyObject_VAR_HEAD + Py_hash_t ob_shash; + char ob_sval[1]; + + /* Invariants: + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. + * ob_shash is the hash of the string or -1 if not computed yet. + */ +} PyBytesObject; + +PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); +PyAPI_FUNC(PyObject*) _PyBytes_FormatEx( + const char *format, + Py_ssize_t format_len, + PyObject *args, + int use_bytearray); +PyAPI_FUNC(PyObject*) _PyBytes_FromHex( + PyObject *string, + int use_bytearray); + +/* Helper for PyBytes_DecodeEscape that detects invalid escape chars. */ +PyAPI_FUNC(PyObject *) _PyBytes_DecodeEscape(const char *, Py_ssize_t, + const char *, const char **); + +/* Macro, trading safety for speed */ +#define PyBytes_AS_STRING(op) (assert(PyBytes_Check(op)), \ + (((PyBytesObject *)(op))->ob_sval)) +#define PyBytes_GET_SIZE(op) (assert(PyBytes_Check(op)),Py_SIZE(op)) + +/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, + x must be an iterable object. */ +PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); + + +/* The _PyBytesWriter structure is big: it contains an embedded "stack buffer". + A _PyBytesWriter variable must be declared at the end of variables in a + function to optimize the memory allocation on the stack. */ +typedef struct { + /* bytes, bytearray or NULL (when the small buffer is used) */ + PyObject *buffer; + + /* Number of allocated size. */ + Py_ssize_t allocated; + + /* Minimum number of allocated bytes, + incremented by _PyBytesWriter_Prepare() */ + Py_ssize_t min_size; + + /* If non-zero, use a bytearray instead of a bytes object for buffer. */ + int use_bytearray; + + /* If non-zero, overallocate the buffer (default: 0). + This flag must be zero if use_bytearray is non-zero. */ + int overallocate; + + /* Stack buffer */ + int use_small_buffer; + char small_buffer[512]; +} _PyBytesWriter; + +/* Initialize a bytes writer + + By default, the overallocation is disabled. Set the overallocate attribute + to control the allocation of the buffer. */ +PyAPI_FUNC(void) _PyBytesWriter_Init(_PyBytesWriter *writer); + +/* Get the buffer content and reset the writer. + Return a bytes object, or a bytearray object if use_bytearray is non-zero. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(PyObject *) _PyBytesWriter_Finish(_PyBytesWriter *writer, + void *str); + +/* Deallocate memory of a writer (clear its internal buffer). */ +PyAPI_FUNC(void) _PyBytesWriter_Dealloc(_PyBytesWriter *writer); + +/* Allocate the buffer to write size bytes. + Return the pointer to the beginning of buffer data. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(void*) _PyBytesWriter_Alloc(_PyBytesWriter *writer, + Py_ssize_t size); + +/* Ensure that the buffer is large enough to write *size* bytes. + Add size to the writer minimum size (min_size attribute). + + str is the current pointer inside the buffer. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(void*) _PyBytesWriter_Prepare(_PyBytesWriter *writer, + void *str, + Py_ssize_t size); + +/* Resize the buffer to make it larger. + The new buffer may be larger than size bytes because of overallocation. + Return the updated current pointer inside the buffer. + Raise an exception and return NULL on error. + + Note: size must be greater than the number of allocated bytes in the writer. + + This function doesn't use the writer minimum size (min_size attribute). + + See also _PyBytesWriter_Prepare(). + */ +PyAPI_FUNC(void*) _PyBytesWriter_Resize(_PyBytesWriter *writer, + void *str, + Py_ssize_t size); + +/* Write bytes. + Raise an exception and return NULL on error. */ +PyAPI_FUNC(void*) _PyBytesWriter_WriteBytes(_PyBytesWriter *writer, + void *str, + const void *bytes, + Py_ssize_t size); diff --git a/Makefile.pre.in b/Makefile.pre.in index f5540a2f0a6b4..2f3fab7b33ccb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1055,6 +1055,8 @@ PYTHON_HEADERS= \ $(srcdir)/Include/Python-ast.h \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/bytearrayobject.h \ + $(srcdir)/Include/cpython/bytesobject.h \ $(srcdir)/Include/cpython/ceval.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7d597bcdac666..9e6323fdcbb0d 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -126,6 +126,8 @@ <ClInclude Include="..\Include\complexobject.h" /> <ClInclude Include="..\Include\context.h" /> <ClInclude Include="..\Include\cpython\abstract.h" /> + <ClInclude Include="..\Include\cpython\bytearrayobject.h" /> + <ClInclude Include="..\Include\cpython\bytesobject.h" /> <ClInclude Include="..\Include\cpython\ceval.h" /> <ClInclude Include="..\Include\cpython\dictobject.h" /> <ClInclude Include="..\Include\cpython\fileobject.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 9563bdc25ebdb..b1412d550c05a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -81,6 +81,12 @@ <ClInclude Include="..\Include\cpython\abstract.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\cpython\bytearrayobject.h"> + <Filter>Include</Filter> + </ClInclude> + <ClInclude Include="..\Include\cpython\bytesobject.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\cpython\ceval.h"> <Filter>Include</Filter> </ClInclude>

1 0

bpo-35081: Move dtoa.h header to the internal C API (GH-18489)
by Victor Stinner 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/e9e7d284c434768333fdfb53a3663eae74… commit: e9e7d284c434768333fdfb53a3663eae74cb995a branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-12T22:54:42+01:00 summary: bpo-35081: Move dtoa.h header to the internal C API (GH-18489) Move the dtoa.h header file to the internal C API as pycore_dtoa.h: it only contains private functions (prefixed by "_Py"). The math and cmath modules must now be compiled with the Py_BUILD_CORE macro defined. files: A Include/internal/pycore_dtoa.h A Misc/NEWS.d/next/C API/2020-02-12-21-24-02.bpo-35081.at7BjN.rst D Include/dtoa.h M Include/Python.h M Makefile.pre.in M Modules/Setup M Modules/cmathmodule.c M Modules/mathmodule.c M Objects/floatobject.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters M Python/dtoa.c M Python/pystrtod.c M setup.py diff --git a/Include/Python.h b/Include/Python.h index d6e5b139ac679..969d8e6bea741 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -152,7 +152,6 @@ #include "pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" -#include "dtoa.h" #include "fileutils.h" #include "pyfpe.h" #include "tracemalloc.h" diff --git a/Include/dtoa.h b/Include/internal/pycore_dtoa.h similarity index 66% rename from Include/dtoa.h rename to Include/internal/pycore_dtoa.h index 9bfb6251db831..3faf8cf6b2eef 100644 --- a/Include/dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -1,9 +1,15 @@ -#ifndef Py_LIMITED_API #ifndef PY_NO_SHORT_FLOAT_REPR #ifdef __cplusplus extern "C" { #endif +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +/* These functions are used by modules compiled as C extension like math: + they must be exported. */ + PyAPI_FUNC(double) _Py_dg_strtod(const char *str, char **ptr); PyAPI_FUNC(char *) _Py_dg_dtoa(double d, int mode, int ndigits, int *decpt, int *sign, char **rve); @@ -11,9 +17,7 @@ PyAPI_FUNC(void) _Py_dg_freedtoa(char *s); PyAPI_FUNC(double) _Py_dg_stdnan(int sign); PyAPI_FUNC(double) _Py_dg_infinity(int sign); - #ifdef __cplusplus } #endif -#endif -#endif +#endif /* !PY_NO_SHORT_FLOAT_REPR */ diff --git a/Makefile.pre.in b/Makefile.pre.in index aae93ff82c145..f5540a2f0a6b4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -981,7 +981,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/context.h \ $(srcdir)/Include/descrobject.h \ $(srcdir)/Include/dictobject.h \ - $(srcdir)/Include/dtoa.h \ $(srcdir)/Include/dynamic_annotations.h \ $(srcdir)/Include/enumobject.h \ $(srcdir)/Include/errcode.h \ @@ -1082,6 +1081,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_code.h \ $(srcdir)/Include/internal/pycore_condvar.h \ $(srcdir)/Include/internal/pycore_context.h \ + $(srcdir)/Include/internal/pycore_dtoa.h \ $(srcdir)/Include/internal/pycore_fileutils.h \ $(srcdir)/Include/internal/pycore_getopt.h \ $(srcdir)/Include/internal/pycore_gil.h \ diff --git a/Misc/NEWS.d/next/C API/2020-02-12-21-24-02.bpo-35081.at7BjN.rst b/Misc/NEWS.d/next/C API/2020-02-12-21-24-02.bpo-35081.at7BjN.rst new file mode 100644 index 0000000000000..94e6ae7e42cc8 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-02-12-21-24-02.bpo-35081.at7BjN.rst @@ -0,0 +1,5 @@ +Move the ``dtoa.h`` header file to the internal C API as ``pycore_dtoa.h``: +it only contains private functions (prefixed by ``_Py``). The :mod:`math` and +:mod:`cmath` modules must now be compiled with the ``Py_BUILD_CORE`` macro +defined. + diff --git a/Modules/Setup b/Modules/Setup index 983fa014ecb24..40266a192bc5e 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -167,8 +167,8 @@ _symtable symtablemodule.c # Modules that should always be present (non UNIX dependent): #array arraymodule.c # array objects -#cmath cmathmodule.c _math.c # -lm # complex math library functions -#math mathmodule.c _math.c # -lm # math library functions, e.g. sin() +#cmath cmathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # complex math library functions +#math mathmodule.c _math.c -DPy_BUILD_CORE_MODULE # -lm # math library functions, e.g. sin() #_contextvars _contextvarsmodule.c # Context Variables #_struct _struct.c # binary structure packing/unpacking #_weakref _weakref.c # basic weak reference support diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 8b21decfa53fc..5eac4b4940bea 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -3,6 +3,7 @@ /* much code borrowed from mathmodule.c */ #include "Python.h" +#include "pycore_dtoa.h" #include "_math.h" /* we need DBL_MAX, DBL_MIN, DBL_EPSILON, DBL_MANT_DIG and FLT_RADIX from float.h. We assume that FLT_RADIX is either 2 or 16. */ diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index f012b51d86698..309f229159540 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -53,6 +53,7 @@ raised for division by zero and mod by zero. */ #include "Python.h" +#include "pycore_dtoa.h" #include "_math.h" #include "clinic/mathmodule.c.h" diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 648030b659c23..04f968e56b142 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -4,6 +4,7 @@ for any kind of float exception without losing portability. */ #include "Python.h" +#include "pycore_dtoa.h" #include <ctype.h> #include <float.h> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index a3719d8558de7..7d597bcdac666 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -166,6 +166,7 @@ <ClInclude Include="..\Include\internal\pycore_code.h" /> <ClInclude Include="..\Include\internal\pycore_condvar.h" /> <ClInclude Include="..\Include\internal\pycore_context.h" /> + <ClInclude Include="..\Include\internal\pycore_dtoa.h" /> <ClInclude Include="..\Include\internal\pycore_fileutils.h" /> <ClInclude Include="..\Include\internal\pycore_getopt.h" /> <ClInclude Include="..\Include\internal\pycore_gil.h" /> @@ -223,7 +224,6 @@ <ClInclude Include="..\Include\pystrcmp.h" /> <ClInclude Include="..\Include\pystrtod.h" /> <ClInclude Include="..\Include\pystrhex.h" /> - <ClInclude Include="..\Include\dtoa.h" /> <ClInclude Include="..\Include\Python-ast.h" /> <ClInclude Include="..\Include\Python.h" /> <ClInclude Include="..\Include\pythonrun.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 67e223dab4396..9563bdc25ebdb 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -201,6 +201,9 @@ <ClInclude Include="..\Include\internal\pycore_context.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_dtoa.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_fileutils.h"> <Filter>Include</Filter> </ClInclude> @@ -360,9 +363,6 @@ <ClInclude Include="..\Include\pystrhex.h"> <Filter>Include</Filter> </ClInclude> - <ClInclude Include="..\Include\dtoa.h"> - <Filter>Include</Filter> - </ClInclude> <ClInclude Include="..\Include\Python-ast.h"> <Filter>Include</Filter> </ClInclude> diff --git a/Python/dtoa.c b/Python/dtoa.c index b7bb7acfb6c21..822adc612962a 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -115,6 +115,7 @@ /* Linking of Python's #defines to Gay's #defines starts here. */ #include "Python.h" +#include "pycore_dtoa.h" /* if PY_NO_SHORT_FLOAT_REPR is defined, then don't even try to compile the following code */ diff --git a/Python/pystrtod.c b/Python/pystrtod.c index 94dc4818c2f47..1c8202c776188 100644 --- a/Python/pystrtod.c +++ b/Python/pystrtod.c @@ -1,6 +1,7 @@ /* -*- Mode: C; c-file-style: "python" -*- */ #include <Python.h> +#include "pycore_dtoa.h" #include <locale.h> /* Case-insensitive string match used for nan and inf detection; t should be diff --git a/setup.py b/setup.py index 02f523c42d355..51e67fe4a558b 100644 --- a/setup.py +++ b/setup.py @@ -734,12 +734,14 @@ def detect_simple_extensions(self): # math library functions, e.g. sin() self.add(Extension('math', ['mathmodule.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], extra_objects=[shared_math], depends=['_math.h', shared_math], libraries=['m'])) # complex math library functions self.add(Extension('cmath', ['cmathmodule.c'], + extra_compile_args=['-DPy_BUILD_CORE_MODULE'], extra_objects=[shared_math], depends=['_math.h', shared_math], libraries=['m']))

1 0

bpo-35081: Move bytes_methods.h to the internal C API (GH-18492)
by Victor Stinner 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/45876a90e2663f12b90c2090ec3e48bd97… commit: 45876a90e2663f12b90c2090ec3e48bd97841aae branch: master author: Victor Stinner <vstinner(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-12T22:32:34+01:00 summary: bpo-35081: Move bytes_methods.h to the internal C API (GH-18492) Move the bytes_methods.h header file to the internal C API as pycore_bytes_methods.h: it only contains private symbols (prefixed by "_Py"), except of the PyDoc_STRVAR_shared() macro. files: A Include/internal/pycore_bytes_methods.h A Misc/NEWS.d/next/C API/2020-02-12-21-38-49.bpo-35081.5tj1yC.rst D Include/bytes_methods.h M Makefile.pre.in M Objects/bytearrayobject.c M Objects/bytes_methods.c M Objects/bytesobject.c M Objects/stringlib/ctype.h M Objects/unicodeobject.c M PCbuild/pythoncore.vcxproj M PCbuild/pythoncore.vcxproj.filters diff --git a/Include/bytes_methods.h b/Include/internal/pycore_bytes_methods.h similarity index 97% rename from Include/bytes_methods.h rename to Include/internal/pycore_bytes_methods.h index 8434a50a4bba7..11e8ab20e9136 100644 --- a/Include/bytes_methods.h +++ b/Include/internal/pycore_bytes_methods.h @@ -2,6 +2,10 @@ #ifndef Py_BYTES_CTYPE_H #define Py_BYTES_CTYPE_H +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + /* * The internal implementation behind PyBytes (bytes) and PyByteArray (bytearray) * methods of the given names, they operate on ASCII byte strings. diff --git a/Makefile.pre.in b/Makefile.pre.in index 3da104bac87d0..aae93ff82c145 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -970,7 +970,6 @@ PYTHON_HEADERS= \ $(srcdir)/Include/bltinmodule.h \ $(srcdir)/Include/boolobject.h \ $(srcdir)/Include/bytearrayobject.h \ - $(srcdir)/Include/bytes_methods.h \ $(srcdir)/Include/bytesobject.h \ $(srcdir)/Include/cellobject.h \ $(srcdir)/Include/ceval.h \ @@ -1077,6 +1076,7 @@ PYTHON_HEADERS= \ \ $(srcdir)/Include/internal/pycore_accu.h \ $(srcdir)/Include/internal/pycore_atomic.h \ + $(srcdir)/Include/internal/pycore_bytes_methods.h \ $(srcdir)/Include/internal/pycore_call.h \ $(srcdir)/Include/internal/pycore_ceval.h \ $(srcdir)/Include/internal/pycore_code.h \ diff --git a/Misc/NEWS.d/next/C API/2020-02-12-21-38-49.bpo-35081.5tj1yC.rst b/Misc/NEWS.d/next/C API/2020-02-12-21-38-49.bpo-35081.5tj1yC.rst new file mode 100644 index 0000000000000..6be33200d9e2b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-02-12-21-38-49.bpo-35081.5tj1yC.rst @@ -0,0 +1,3 @@ +Move the ``bytes_methods.h`` header file to the internal C API as +``pycore_bytes_methods.h``: it only contains private symbols (prefixed by +``_Py``), except of the ``PyDoc_STRVAR_shared()`` macro. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index a3fc35ca4d22a..d3964358bc59d 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2,11 +2,11 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_bytes_methods.h" #include "pycore_object.h" #include "pycore_pymem.h" #include "pycore_pystate.h" #include "structmember.h" -#include "bytes_methods.h" #include "bytesobject.h" #include "pystrhex.h" diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index db030be4fe756..a4b3868e72522 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -1,6 +1,6 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" -#include "bytes_methods.h" +#include "pycore_bytes_methods.h" PyDoc_STRVAR_shared(_Py_isspace__doc__, "B.isspace() -> bool\n\ diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index df3eddae6a36d..bd8af72ade5d3 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3,11 +3,11 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_bytes_methods.h" #include "pycore_object.h" #include "pycore_pymem.h" #include "pycore_pystate.h" -#include "bytes_methods.h" #include "pystrhex.h" #include <stddef.h> diff --git a/Objects/stringlib/ctype.h b/Objects/stringlib/ctype.h index 843cfa22a8454..9b319b07d11bc 100644 --- a/Objects/stringlib/ctype.h +++ b/Objects/stringlib/ctype.h @@ -2,7 +2,7 @@ # error "ctype.h only compatible with byte-wise strings" #endif -#include "bytes_methods.h" +#include "pycore_bytes_methods.h" static PyObject* stringlib_isspace(PyObject *self, PyObject *Py_UNUSED(ignored)) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 8470e41624bd4..11fa1fb5ff798 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -40,6 +40,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "pycore_bytes_methods.h" #include "pycore_fileutils.h" #include "pycore_initconfig.h" #include "pycore_object.h" @@ -47,7 +48,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pylifecycle.h" #include "pycore_pystate.h" #include "ucnhash.h" -#include "bytes_methods.h" #include "stringlib/eq.h" #ifdef MS_WINDOWS diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 36a27f467405d..a3719d8558de7 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -115,7 +115,6 @@ <ClInclude Include="..\Include\ast.h" /> <ClInclude Include="..\Include\bitset.h" /> <ClInclude Include="..\Include\boolobject.h" /> - <ClInclude Include="..\Include\bytes_methods.h" /> <ClInclude Include="..\Include\bytearrayobject.h" /> <ClInclude Include="..\Include\bytesobject.h" /> <ClInclude Include="..\Include\cellobject.h" /> @@ -161,6 +160,7 @@ <ClInclude Include="..\Include\import.h" /> <ClInclude Include="..\Include\internal\pycore_accu.h" /> <ClInclude Include="..\Include\internal\pycore_atomic.h" /> + <ClInclude Include="..\Include\internal\pycore_bytes_methods.h" /> <ClInclude Include="..\Include\internal\pycore_call.h" /> <ClInclude Include="..\Include\internal\pycore_ceval.h" /> <ClInclude Include="..\Include\internal\pycore_code.h" /> diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0301557b3e50d..67e223dab4396 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -48,9 +48,6 @@ <ClInclude Include="..\Include\boolobject.h"> <Filter>Include</Filter> </ClInclude> - <ClInclude Include="..\Include\bytes_methods.h"> - <Filter>Include</Filter> - </ClInclude> <ClInclude Include="..\Include\bytearrayobject.h"> <Filter>Include</Filter> </ClInclude> @@ -186,6 +183,9 @@ <ClInclude Include="..\Include\internal\pycore_atomic.h"> <Filter>Include</Filter> </ClInclude> + <ClInclude Include="..\Include\internal\pycore_bytes_methods.h"> + <Filter>Include</Filter> + </ClInclude> <ClInclude Include="..\Include\internal\pycore_call.h"> <Filter>Include</Filter> </ClInclude>

1 0

bpo-39474: Fix AST pos for expressions like (a)(b), (a)[b] and (a).b. (GH-18477)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/2076d4f97ef514bb4dc4ca768fbaa3f538… commit: 2076d4f97ef514bb4dc4ca768fbaa3f538ce7f1f branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T12:56:44-08:00 summary: bpo-39474: Fix AST pos for expressions like (a)(b), (a)[b] and (a).b. (GH-18477) (cherry picked from commit 6e619c48b8e804ece9521453fc8da0640a04d5b1) Co-authored-by: Serhiy Storchaka <storchaka(a)gmail.com> files: A Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst M Lib/test/test_ast.py M Python/ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index e843d53781d25..3e8a39dc41047 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1642,6 +1642,33 @@ def test_attribute_spaces(self): self._check_content(s, call, s) self._check_content(s, call.args[0], 'x. y .z') + def test_redundant_parenthesis(self): + s = '( ( ( a + b ) ) )' + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s, v, 'a + b') + s2 = 'await ' + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s2, v, 'a + b') + + def test_trailers_with_redundant_parenthesis(self): + tests = ( + ('( ( ( a ) ) ) ( )', 'Call'), + ('( ( ( a ) ) ) ( b )', 'Call'), + ('( ( ( a ) ) ) [ b ]', 'Subscript'), + ('( ( ( a ) ) ) . b', 'Attribute'), + ) + for s, t in tests: + with self.subTest(s): + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, t) + self._check_content(s, v, s) + s2 = 'await ' + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, t) + self._check_content(s2, v, s) + def test_displays(self): s1 = '[{}, {1, }, {1, 2,} ]' s2 = '{a: b, f (): g () ,}' diff --git a/Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst b/Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst new file mode 100644 index 0000000000000..e990f84a9dbf2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst @@ -0,0 +1,2 @@ +Fixed starting position of AST for expressions like ``(a)(b)``, ``(a)[b]`` +and ``(a).b``. diff --git a/Python/ast.c b/Python/ast.c index 12f24f2c22ab9..f70d48ba3a15d 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -583,7 +583,7 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, bool); /* Note different signature for ast_for_call */ static expr_ty ast_for_call(struct compiling *, const node *, expr_ty, - const node *, const node *); + const node *, const node *, const node *); static PyObject *parsenumber(struct compiling *, const char *); static expr_ty parsestrplus(struct compiling *, const node *n); @@ -1757,7 +1757,8 @@ ast_for_decorator(struct compiling *c, const node *n) name_expr = NULL; } else { - d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2), CHILD(n, 4)); + d = ast_for_call(c, CHILD(n, 3), name_expr, + CHILD(n, 1), CHILD(n, 2), CHILD(n, 4)); if (!d) return NULL; name_expr = NULL; @@ -2658,7 +2659,7 @@ ast_for_binop(struct compiling *c, const node *n) } static expr_ty -ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) +ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr, const node *start) { /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] @@ -2668,17 +2669,18 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) REQ(n, trailer); if (TYPE(CHILD(n, 0)) == LPAR) { if (NCH(n) == 2) - return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset, + return Call(left_expr, NULL, NULL, LINENO(start), start->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else - return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0), CHILD(n, 2)); + return ast_for_call(c, CHILD(n, 1), left_expr, + start, CHILD(n, 0), CHILD(n, 2)); } else if (TYPE(CHILD(n, 0)) == DOT) { PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1)); if (!attr_id) return NULL; return Attribute(left_expr, attr_id, Load, - LINENO(n), n->n_col_offset, + LINENO(start), start->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { @@ -2689,7 +2691,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) slice_ty slc = ast_for_slice(c, CHILD(n, 0)); if (!slc) return NULL; - return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, + return Subscript(left_expr, slc, Load, LINENO(start), start->n_col_offset, n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } @@ -2716,7 +2718,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) } if (!simple) { return Subscript(left_expr, ExtSlice(slices, c->c_arena), - Load, LINENO(n), n->n_col_offset, + Load, LINENO(start), start->n_col_offset, n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } /* extract Index values and put them in a Tuple */ @@ -2733,7 +2735,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!e) return NULL; return Subscript(left_expr, Index(e, c->c_arena), - Load, LINENO(n), n->n_col_offset, + Load, LINENO(start), start->n_col_offset, n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } } @@ -2771,7 +2773,7 @@ static expr_ty ast_for_atom_expr(struct compiling *c, const node *n) { int i, nch, start = 0; - expr_ty e, tmp; + expr_ty e; REQ(n, atom_expr); nch = NCH(n); @@ -2800,12 +2802,9 @@ ast_for_atom_expr(struct compiling *c, const node *n) node *ch = CHILD(n, i); if (TYPE(ch) != trailer) break; - tmp = ast_for_trailer(c, ch, e); - if (!tmp) + e = ast_for_trailer(c, ch, e, CHILD(n, start)); + if (!e) return NULL; - tmp->lineno = e->lineno; - tmp->col_offset = e->col_offset; - e = tmp; } if (start) { @@ -3035,7 +3034,7 @@ ast_for_expr(struct compiling *c, const node *n) static expr_ty ast_for_call(struct compiling *c, const node *n, expr_ty func, - const node *maybegenbeg, const node *closepar) + const node *start, const node *maybegenbeg, const node *closepar) { /* arglist: argument (',' argument)* [','] @@ -3239,7 +3238,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, } } - return Call(func, args, keywords, func->lineno, func->col_offset, + return Call(func, args, keywords, LINENO(start), start->n_col_offset, closepar->n_end_lineno, closepar->n_end_col_offset, c->c_arena); } @@ -4489,7 +4488,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, c->c_arena); - call = ast_for_call(c, CHILD(n, 3), dummy, NULL, CHILD(n, 4)); + call = ast_for_call(c, CHILD(n, 3), dummy, + CHILD(n, 1), NULL, CHILD(n, 4)); if (!call) return NULL; }

1 0

bpo-39474: Fix AST pos for expressions like (a)(b), (a)[b] and (a).b. (GH-18477)
by Serhiy Storchaka 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/6e619c48b8e804ece9521453fc8da0640a… commit: 6e619c48b8e804ece9521453fc8da0640a04d5b1 branch: master author: Serhiy Storchaka <storchaka(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T22:37:49+02:00 summary: bpo-39474: Fix AST pos for expressions like (a)(b), (a)[b] and (a).b. (GH-18477) files: A Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst M Lib/test/test_ast.py M Python/ast.c diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 6d1e419932261..2ed4657822e54 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1707,6 +1707,33 @@ def test_attribute_spaces(self): self._check_content(s, call, s) self._check_content(s, call.args[0], 'x. y .z') + def test_redundant_parenthesis(self): + s = '( ( ( a + b ) ) )' + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s, v, 'a + b') + s2 = 'await ' + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, 'BinOp') + self._check_content(s2, v, 'a + b') + + def test_trailers_with_redundant_parenthesis(self): + tests = ( + ('( ( ( a ) ) ) ( )', 'Call'), + ('( ( ( a ) ) ) ( b )', 'Call'), + ('( ( ( a ) ) ) [ b ]', 'Subscript'), + ('( ( ( a ) ) ) . b', 'Attribute'), + ) + for s, t in tests: + with self.subTest(s): + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, t) + self._check_content(s, v, s) + s2 = 'await ' + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, t) + self._check_content(s2, v, s) + def test_displays(self): s1 = '[{}, {1, }, {1, 2,} ]' s2 = '{a: b, f (): g () ,}' diff --git a/Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst b/Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst new file mode 100644 index 0000000000000..e990f84a9dbf2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-12-12-01-26.bpo-39474.RZMEUH.rst @@ -0,0 +1,2 @@ +Fixed starting position of AST for expressions like ``(a)(b)``, ``(a)[b]`` +and ``(a).b``. diff --git a/Python/ast.c b/Python/ast.c index bab672b29589f..ad25565b7c7f8 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -583,7 +583,7 @@ static stmt_ty ast_for_for_stmt(struct compiling *, const node *, bool); /* Note different signature for ast_for_call */ static expr_ty ast_for_call(struct compiling *, const node *, expr_ty, - const node *, const node *); + const node *, const node *, const node *); static PyObject *parsenumber(struct compiling *, const char *); static expr_ty parsestrplus(struct compiling *, const node *n); @@ -1757,7 +1757,8 @@ ast_for_decorator(struct compiling *c, const node *n) name_expr = NULL; } else { - d = ast_for_call(c, CHILD(n, 3), name_expr, CHILD(n, 2), CHILD(n, 4)); + d = ast_for_call(c, CHILD(n, 3), name_expr, + CHILD(n, 1), CHILD(n, 2), CHILD(n, 4)); if (!d) return NULL; name_expr = NULL; @@ -2658,7 +2659,7 @@ ast_for_binop(struct compiling *c, const node *n) } static expr_ty -ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) +ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr, const node *start) { /* trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] @@ -2668,17 +2669,18 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) REQ(n, trailer); if (TYPE(CHILD(n, 0)) == LPAR) { if (NCH(n) == 2) - return Call(left_expr, NULL, NULL, LINENO(n), n->n_col_offset, + return Call(left_expr, NULL, NULL, LINENO(start), start->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); else - return ast_for_call(c, CHILD(n, 1), left_expr, CHILD(n, 0), CHILD(n, 2)); + return ast_for_call(c, CHILD(n, 1), left_expr, + start, CHILD(n, 0), CHILD(n, 2)); } else if (TYPE(CHILD(n, 0)) == DOT) { PyObject *attr_id = NEW_IDENTIFIER(CHILD(n, 1)); if (!attr_id) return NULL; return Attribute(left_expr, attr_id, Load, - LINENO(n), n->n_col_offset, + LINENO(start), start->n_col_offset, n->n_end_lineno, n->n_end_col_offset, c->c_arena); } else { @@ -2689,7 +2691,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) slice_ty slc = ast_for_slice(c, CHILD(n, 0)); if (!slc) return NULL; - return Subscript(left_expr, slc, Load, LINENO(n), n->n_col_offset, + return Subscript(left_expr, slc, Load, LINENO(start), start->n_col_offset, n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } @@ -2716,7 +2718,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) } if (!simple) { return Subscript(left_expr, ExtSlice(slices, c->c_arena), - Load, LINENO(n), n->n_col_offset, + Load, LINENO(start), start->n_col_offset, n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } /* extract Index values and put them in a Tuple */ @@ -2733,7 +2735,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr) if (!e) return NULL; return Subscript(left_expr, Index(e, c->c_arena), - Load, LINENO(n), n->n_col_offset, + Load, LINENO(start), start->n_col_offset, n_copy->n_end_lineno, n_copy->n_end_col_offset, c->c_arena); } } @@ -2771,7 +2773,7 @@ static expr_ty ast_for_atom_expr(struct compiling *c, const node *n) { int i, nch, start = 0; - expr_ty e, tmp; + expr_ty e; REQ(n, atom_expr); nch = NCH(n); @@ -2800,12 +2802,9 @@ ast_for_atom_expr(struct compiling *c, const node *n) node *ch = CHILD(n, i); if (TYPE(ch) != trailer) break; - tmp = ast_for_trailer(c, ch, e); - if (!tmp) + e = ast_for_trailer(c, ch, e, CHILD(n, start)); + if (!e) return NULL; - tmp->lineno = e->lineno; - tmp->col_offset = e->col_offset; - e = tmp; } if (start) { @@ -3035,7 +3034,7 @@ ast_for_expr(struct compiling *c, const node *n) static expr_ty ast_for_call(struct compiling *c, const node *n, expr_ty func, - const node *maybegenbeg, const node *closepar) + const node *start, const node *maybegenbeg, const node *closepar) { /* arglist: argument (',' argument)* [','] @@ -3239,7 +3238,7 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func, } } - return Call(func, args, keywords, func->lineno, func->col_offset, + return Call(func, args, keywords, LINENO(start), start->n_col_offset, closepar->n_end_lineno, closepar->n_end_col_offset, c->c_arena); } @@ -4486,7 +4485,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq) dummy = Name(dummy_name, Load, LINENO(n), n->n_col_offset, CHILD(n, 1)->n_end_lineno, CHILD(n, 1)->n_end_col_offset, c->c_arena); - call = ast_for_call(c, CHILD(n, 3), dummy, NULL, CHILD(n, 4)); + call = ast_for_call(c, CHILD(n, 3), dummy, + CHILD(n, 1), NULL, CHILD(n, 4)); if (!call) return NULL; }

1 0

bpo-18819: tarfile: only set device fields for device files (GH-18080)
by William Chargin 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/674935b8caf33e47c78f1b8e197b1b77a0… commit: 674935b8caf33e47c78f1b8e197b1b77a04992d2 branch: master author: William Chargin <wchargin(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T11:56:02-08:00 summary: bpo-18819: tarfile: only set device fields for device files (GH-18080) The GNU docs describe the `devmajor` and `devminor` fields of the tar header struct only in the context of character and block special files, suggesting that in other cases they are not populated. Typical utilities behave accordingly; this patch teaches `tarfile` to do the same. files: A Misc/NEWS.d/next/Library/2020-01-20-10-06-19.bpo-18819.H4qsoS.rst M Lib/tarfile.py M Lib/test/test_tarfile.py M Misc/ACKS diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 90a2c95b315b3..e2b60532f693d 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -930,6 +930,14 @@ def _create_header(info, format, encoding, errors): """Return a header block. info is a dictionary with file information, format must be one of the *_FORMAT constants. """ + has_device_fields = info.get("type") in (CHRTYPE, BLKTYPE) + if has_device_fields: + devmajor = itn(info.get("devmajor", 0), 8, format) + devminor = itn(info.get("devminor", 0), 8, format) + else: + devmajor = stn("", 8, encoding, errors) + devminor = stn("", 8, encoding, errors) + parts = [ stn(info.get("name", ""), 100, encoding, errors), itn(info.get("mode", 0) & 0o7777, 8, format), @@ -943,8 +951,8 @@ def _create_header(info, format, encoding, errors): info.get("magic", POSIX_MAGIC), stn(info.get("uname", ""), 32, encoding, errors), stn(info.get("gname", ""), 32, encoding, errors), - itn(info.get("devmajor", 0), 8, format), - itn(info.get("devminor", 0), 8, format), + devmajor, + devminor, stn(info.get("prefix", ""), 155, encoding, errors) ] diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 6a901089611cd..cae96802ded67 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1549,6 +1549,52 @@ def test_longnamelink_1025(self): ("longlnk/" * 127) + "longlink_") +class DeviceHeaderTest(WriteTestBase, unittest.TestCase): + + prefix = "w:" + + def test_headers_written_only_for_device_files(self): + # Regression test for bpo-18819. + tempdir = os.path.join(TEMPDIR, "device_header_test") + os.mkdir(tempdir) + try: + tar = tarfile.open(tmpname, self.mode) + try: + input_blk = tarfile.TarInfo(name="my_block_device") + input_reg = tarfile.TarInfo(name="my_regular_file") + input_blk.type = tarfile.BLKTYPE + input_reg.type = tarfile.REGTYPE + tar.addfile(input_blk) + tar.addfile(input_reg) + finally: + tar.close() + + # devmajor and devminor should be *interpreted* as 0 in both... + tar = tarfile.open(tmpname, "r") + try: + output_blk = tar.getmember("my_block_device") + output_reg = tar.getmember("my_regular_file") + finally: + tar.close() + self.assertEqual(output_blk.devmajor, 0) + self.assertEqual(output_blk.devminor, 0) + self.assertEqual(output_reg.devmajor, 0) + self.assertEqual(output_reg.devminor, 0) + + # ...but the fields should not actually be set on regular files: + with open(tmpname, "rb") as infile: + buf = infile.read() + buf_blk = buf[output_blk.offset:output_blk.offset_data] + buf_reg = buf[output_reg.offset:output_reg.offset_data] + # See `struct posixheader` in GNU docs for byte offsets: + # <https://www.gnu.org/software/tar/manual/html_node/Standard.html> + device_headers = slice(329, 329 + 16) + self.assertEqual(buf_blk[device_headers], b"0000000\0" * 2) + self.assertEqual(buf_reg[device_headers], b"\0" * 16) + finally: + support.rmtree(tempdir) + + class CreateTest(WriteTestBase, unittest.TestCase): prefix = "x:" diff --git a/Misc/ACKS b/Misc/ACKS index 5a779833e68be..933402069b4fd 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -286,6 +286,7 @@ Brad Chapman Greg Chapman Mitch Chapman Matt Chaput +William Chargin Yogesh Chaudhari David Chaum Nicolas Chauvat diff --git a/Misc/NEWS.d/next/Library/2020-01-20-10-06-19.bpo-18819.H4qsoS.rst b/Misc/NEWS.d/next/Library/2020-01-20-10-06-19.bpo-18819.H4qsoS.rst new file mode 100644 index 0000000000000..e9f111ad62e28 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-01-20-10-06-19.bpo-18819.H4qsoS.rst @@ -0,0 +1,3 @@ +Omit ``devmajor`` and ``devminor`` fields for non-device files in +:mod:`tarfile` archives, enabling bit-for-bit compatibility with GNU +``tar(1)``.

1 0

bpo-21016: pydoc and trace use sysconfig (GH-18476)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/ac6f4d2db703c0ff88e496bcb7b7fe55cf… commit: ac6f4d2db703c0ff88e496bcb7b7fe55cf2ac458 branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T04:32:52-08:00 summary: bpo-21016: pydoc and trace use sysconfig (GH-18476) bpo-21016, bpo-1294959: The pydoc and trace modules now use the sysconfig module to get the path to the Python standard library, to support uncommon installation path like /usr/lib64/python3.9/ on Fedora. Co-Authored-By: Jan Matějek <jmatejek(a)suse.com> (cherry picked from commit 4fac7ed43ebf1771a8fe86fdfe7b9991f3be78cd) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst M Lib/pydoc.py M Lib/trace.py diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 9a22e56686f61..dc3377d68f8ca 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -66,6 +66,7 @@ class or function within a module or module in a package. If the import platform import re import sys +import sysconfig import time import tokenize import urllib.parse @@ -392,9 +393,7 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: diff --git a/Lib/trace.py b/Lib/trace.py index 62325d3f238ad..a44735761df42 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -52,6 +52,7 @@ import linecache import os import sys +import sysconfig import token import tokenize import inspect @@ -676,9 +677,8 @@ def main(): opts = parser.parse_args() if opts.ignore_dir: - rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) - _prefix = os.path.join(sys.base_prefix, *rel_path) - _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + _prefix = sysconfig.get_path("stdlib") + _exec_prefix = sysconfig.get_path("platstdlib") def parse_ignore_dir(s): s = os.path.expanduser(os.path.expandvars(s)) diff --git a/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst b/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst new file mode 100644 index 0000000000000..fb91bb3825555 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst @@ -0,0 +1,4 @@ +The :mod:`pydoc` and :mod:`trace` modules now use the :mod:`sysconfig` +module to get the path to the Python standard library, to support uncommon +installation path like ``/usr/lib64/python3.9/`` on Fedora. +Patch by Jan Matějek.

1 0

bpo-21016: pydoc and trace use sysconfig (GH-18476)
by Miss Islington (bot) 12 Feb '20

12 Feb '20

https://github.com/python/cpython/commit/ca133e53fafdec1aa77613fcb7558deed9… commit: ca133e53fafdec1aa77613fcb7558deed959383f branch: 3.7 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T04:32:46-08:00 summary: bpo-21016: pydoc and trace use sysconfig (GH-18476) bpo-21016, bpo-1294959: The pydoc and trace modules now use the sysconfig module to get the path to the Python standard library, to support uncommon installation path like /usr/lib64/python3.9/ on Fedora. Co-Authored-By: Jan Matějek <jmatejek(a)suse.com> (cherry picked from commit 4fac7ed43ebf1771a8fe86fdfe7b9991f3be78cd) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst M Lib/pydoc.py M Lib/trace.py diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 44df8c854ae9e..978e4cd0baa5b 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -66,6 +66,7 @@ class or function within a module or module in a package. If the import platform import re import sys +import sysconfig import time import tokenize import urllib.parse @@ -398,9 +399,7 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: diff --git a/Lib/trace.py b/Lib/trace.py index 206bd2b689f11..c804a0d75669a 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -53,6 +53,7 @@ import os import re import sys +import sysconfig import token import tokenize import inspect @@ -671,9 +672,8 @@ def main(): opts = parser.parse_args() if opts.ignore_dir: - rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) - _prefix = os.path.join(sys.base_prefix, *rel_path) - _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + _prefix = sysconfig.get_path("stdlib") + _exec_prefix = sysconfig.get_path("platstdlib") def parse_ignore_dir(s): s = os.path.expanduser(os.path.expandvars(s)) diff --git a/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst b/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst new file mode 100644 index 0000000000000..fb91bb3825555 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-12-10-04-39.bpo-21016.bFXPH7.rst @@ -0,0 +1,4 @@ +The :mod:`pydoc` and :mod:`trace` modules now use the :mod:`sysconfig` +module to get the path to the Python standard library, to support uncommon +installation path like ``/usr/lib64/python3.9/`` on Fedora. +Patch by Jan Matějek.

1 0