Mailman 3 February 2020 - Python-checkins

bpo-39627: Fix TypedDict totality check for inherited keys (#18503)
by Vlad Emelianov 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/10e87e5ef4c1b4fb8415d9ddc362e2591f… commit: 10e87e5ef4c1b4fb8415d9ddc362e2591f2f0b6c branch: master author: Vlad Emelianov <volshebnyi(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T11:53:29-08:00 summary: bpo-39627: Fix TypedDict totality check for inherited keys (#18503) (Adapted from https://github.com/python/typing/pull/700) files: A Misc/NEWS.d/next/Library/2020-02-13-18-14-15.bpo-39627.Q0scyQ.rst M Lib/test/test_typing.py M Lib/typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index bc6a3db4e0064..6b0a905048cea 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -3809,6 +3809,38 @@ class Point2Dor3D(Point2D, total=False): assert Point2Dor3D.__required_keys__ == frozenset(['x', 'y']) assert Point2Dor3D.__optional_keys__ == frozenset(['z']) + def test_keys_inheritance(self): + class BaseAnimal(TypedDict): + name: str + + class Animal(BaseAnimal, total=False): + voice: str + tail: bool + + class Cat(Animal): + fur_color: str + + assert BaseAnimal.__required_keys__ == frozenset(['name']) + assert BaseAnimal.__optional_keys__ == frozenset([]) + assert BaseAnimal.__annotations__ == {'name': str} + + assert Animal.__required_keys__ == frozenset(['name']) + assert Animal.__optional_keys__ == frozenset(['tail', 'voice']) + assert Animal.__annotations__ == { + 'name': str, + 'tail': bool, + 'voice': str, + } + + assert Cat.__required_keys__ == frozenset(['name', 'fur_color']) + assert Cat.__optional_keys__ == frozenset(['tail', 'voice']) + assert Cat.__annotations__ == { + 'fur_color': str, + 'name': str, + 'tail': bool, + 'voice': str, + } + class IOTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 8886b08c2ec6f..6da145fcdb83a 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1828,23 +1828,30 @@ def __new__(cls, name, bases, ns, total=True): ns['__new__'] = _typeddict_new if name == 'TypedDict' else _dict_new tp_dict = super(_TypedDictMeta, cls).__new__(cls, name, (dict,), ns) - anns = ns.get('__annotations__', {}) + annotations = {} + own_annotations = ns.get('__annotations__', {}) + own_annotation_keys = set(own_annotations.keys()) msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - anns = {n: _type_check(tp, msg) for n, tp in anns.items()} - required = set(anns if total else ()) - optional = set(() if total else anns) + own_annotations = { + n: _type_check(tp, msg) for n, tp in own_annotations.items() + } + required_keys = set() + optional_keys = set() for base in bases: - base_anns = base.__dict__.get('__annotations__', {}) - anns.update(base_anns) - if getattr(base, '__total__', True): - required.update(base_anns) - else: - optional.update(base_anns) + annotations.update(base.__dict__.get('__annotations__', {})) + required_keys.update(base.__dict__.get('__required_keys__', ())) + optional_keys.update(base.__dict__.get('__optional_keys__', ())) + + annotations.update(own_annotations) + if total: + required_keys.update(own_annotation_keys) + else: + optional_keys.update(own_annotation_keys) - tp_dict.__annotations__ = anns - tp_dict.__required_keys__ = frozenset(required) - tp_dict.__optional_keys__ = frozenset(optional) + tp_dict.__annotations__ = annotations + tp_dict.__required_keys__ = frozenset(required_keys) + tp_dict.__optional_keys__ = frozenset(optional_keys) if not hasattr(tp_dict, '__total__'): tp_dict.__total__ = total return tp_dict diff --git a/Misc/NEWS.d/next/Library/2020-02-13-18-14-15.bpo-39627.Q0scyQ.rst b/Misc/NEWS.d/next/Library/2020-02-13-18-14-15.bpo-39627.Q0scyQ.rst new file mode 100644 index 0000000000000..4806aa67d9535 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-02-13-18-14-15.bpo-39627.Q0scyQ.rst @@ -0,0 +1 @@ +Fixed TypedDict totality check for inherited keys. \ No newline at end of file

1 0

bpo-39524: Fixed doc-string in ast._pad_whitespace (GH-18340)
by mpheath 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/fbeba8f2481411d608a616366394e07cdc… commit: fbeba8f2481411d608a616366394e07cdc52e0bb branch: master author: mpheath <58158242+mpheath(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T20:32:09+02:00 summary: bpo-39524: Fixed doc-string in ast._pad_whitespace (GH-18340) files: M Lib/ast.py diff --git a/Lib/ast.py b/Lib/ast.py index 495c0d618f12c..511f0956a00b0 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -302,7 +302,7 @@ def _splitlines_no_ff(source): def _pad_whitespace(source): - """Replace all chars except '\f\t' in a line with spaces.""" + r"""Replace all chars except '\f\t' in a line with spaces.""" result = '' for c in source: if c in '\f\t':

1 0

bpo-39573: Add Py_IS_TYPE() function (GH-18488)
by Dong-hee Na 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/d905df766c367c350f20c46ccd99d4da19… commit: d905df766c367c350f20c46ccd99d4da19ed57d8 branch: master author: Dong-hee Na <donghee.na92(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T18:37:17+01:00 summary: bpo-39573: Add Py_IS_TYPE() function (GH-18488) Co-Author: Neil Schemenauer <nas-github(a)arctrix.com> files: A Misc/NEWS.d/next/Core and Builtins/2020-02-13-01-30-22.bpo-39573.uTFj1m.rst M Doc/c-api/structures.rst M Include/boolobject.h M Include/bytearrayobject.h M Include/bytesobject.h M Include/cellobject.h M Include/code.h M Include/complexobject.h M Include/context.h M Include/datetime.h M Include/dictobject.h M Include/floatobject.h M Include/funcobject.h M Include/genobject.h M Include/internal/pycore_hamt.h M Include/iterobject.h M Include/listobject.h M Include/memoryobject.h M Include/methodobject.h M Include/moduleobject.h M Include/object.h M Include/odictobject.h M Include/pycapsule.h M Include/rangeobject.h M Include/setobject.h M Include/sliceobject.h M Include/symtable.h M Include/traceback.h M Include/tupleobject.h M Include/unicodeobject.h M Include/weakrefobject.h M Objects/genobject.c diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 083c3740531e4..fc3467bee4d3c 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -70,6 +70,14 @@ the definition of all other Python objects. (((PyObject*)(o))->ob_type) +.. c:function:: int Py_IS_TYPE(PyObject *o, PyTypeObject *type) + + Return non-zero if the object *o* type is *type*. Return zero otherwise. + Equivalent to: ``Py_TYPE(o) == type``. + + .. versionadded:: 3.9 + + .. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) Set the object *o* type to *type*. diff --git a/Include/boolobject.h b/Include/boolobject.h index 7cc2f1fe23937..bb8044a2b02cf 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -9,7 +9,7 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyBool_Type; -#define PyBool_Check(x) (Py_TYPE(x) == &PyBool_Type) +#define PyBool_Check(x) Py_IS_TYPE(x, &PyBool_Type) /* Py_False and Py_True are the only two bools in existence. Don't forget to apply Py_INCREF() when returning either!!! */ diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h index 341ab38a15d5a..9e95433f0f26f 100644 --- a/Include/bytearrayobject.h +++ b/Include/bytearrayobject.h @@ -24,7 +24,7 @@ PyAPI_DATA(PyTypeObject) PyByteArrayIter_Type; /* Type check macros */ #define PyByteArray_Check(self) PyObject_TypeCheck(self, &PyByteArray_Type) -#define PyByteArray_CheckExact(self) (Py_TYPE(self) == &PyByteArray_Type) +#define PyByteArray_CheckExact(self) Py_IS_TYPE(self, &PyByteArray_Type) /* Direct API functions */ PyAPI_FUNC(PyObject *) PyByteArray_FromObject(PyObject *); diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 27c31ee342c88..5062d8d123ad3 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -32,7 +32,7 @@ PyAPI_DATA(PyTypeObject) PyBytesIter_Type; #define PyBytes_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_BYTES_SUBCLASS) -#define PyBytes_CheckExact(op) (Py_TYPE(op) == &PyBytes_Type) +#define PyBytes_CheckExact(op) Py_IS_TYPE(op, &PyBytes_Type) PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); diff --git a/Include/cellobject.h b/Include/cellobject.h index 2f9b5b75d998a..f12aa90a42a8f 100644 --- a/Include/cellobject.h +++ b/Include/cellobject.h @@ -13,7 +13,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyCell_Type; -#define PyCell_Check(op) (Py_TYPE(op) == &PyCell_Type) +#define PyCell_Check(op) Py_IS_TYPE(op, &PyCell_Type) PyAPI_FUNC(PyObject *) PyCell_New(PyObject *); PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *); diff --git a/Include/code.h b/Include/code.h index 3afddd20c80d7..107eba4b9c431 100644 --- a/Include/code.h +++ b/Include/code.h @@ -115,7 +115,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyCode_Type; -#define PyCode_Check(op) (Py_TYPE(op) == &PyCode_Type) +#define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type) #define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars)) /* Public interface */ diff --git a/Include/complexobject.h b/Include/complexobject.h index cb8c52c580085..9221f9c51d65b 100644 --- a/Include/complexobject.h +++ b/Include/complexobject.h @@ -39,7 +39,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyComplex_Type; #define PyComplex_Check(op) PyObject_TypeCheck(op, &PyComplex_Type) -#define PyComplex_CheckExact(op) (Py_TYPE(op) == &PyComplex_Type) +#define PyComplex_CheckExact(op) Py_IS_TYPE(op, &PyComplex_Type) #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex); diff --git a/Include/context.h b/Include/context.h index 9581285247b39..619746d501efd 100644 --- a/Include/context.h +++ b/Include/context.h @@ -17,9 +17,9 @@ PyAPI_DATA(PyTypeObject) PyContextToken_Type; typedef struct _pycontexttokenobject PyContextToken; -#define PyContext_CheckExact(o) (Py_TYPE(o) == &PyContext_Type) -#define PyContextVar_CheckExact(o) (Py_TYPE(o) == &PyContextVar_Type) -#define PyContextToken_CheckExact(o) (Py_TYPE(o) == &PyContextToken_Type) +#define PyContext_CheckExact(o) Py_IS_TYPE(o, &PyContext_Type) +#define PyContextVar_CheckExact(o) Py_IS_TYPE(o, &PyContextVar_Type) +#define PyContextToken_CheckExact(o) Py_IS_TYPE(o, &PyContextToken_Type) PyAPI_FUNC(PyObject *) PyContext_New(void); diff --git a/Include/datetime.h b/Include/datetime.h index 00507cb85cc04..5d9f2558f924d 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -196,19 +196,19 @@ static PyDateTime_CAPI *PyDateTimeAPI = NULL; /* Macros for type checking when not building the Python core. */ #define PyDate_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DateType) -#define PyDate_CheckExact(op) (Py_TYPE(op) == PyDateTimeAPI->DateType) +#define PyDate_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->DateType) #define PyDateTime_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DateTimeType) -#define PyDateTime_CheckExact(op) (Py_TYPE(op) == PyDateTimeAPI->DateTimeType) +#define PyDateTime_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->DateTimeType) #define PyTime_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->TimeType) -#define PyTime_CheckExact(op) (Py_TYPE(op) == PyDateTimeAPI->TimeType) +#define PyTime_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->TimeType) #define PyDelta_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DeltaType) -#define PyDelta_CheckExact(op) (Py_TYPE(op) == PyDateTimeAPI->DeltaType) +#define PyDelta_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->DeltaType) #define PyTZInfo_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->TZInfoType) -#define PyTZInfo_CheckExact(op) (Py_TYPE(op) == PyDateTimeAPI->TZInfoType) +#define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->TZInfoType) /* Macros for accessing constructors in a simplified fashion. */ diff --git a/Include/dictobject.h b/Include/dictobject.h index b37573ad48c00..c88b0aa0a5d0f 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -16,7 +16,7 @@ PyAPI_DATA(PyTypeObject) PyDict_Type; #define PyDict_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS) -#define PyDict_CheckExact(op) (Py_TYPE(op) == &PyDict_Type) +#define PyDict_CheckExact(op) Py_IS_TYPE(op, &PyDict_Type) PyAPI_FUNC(PyObject *) PyDict_New(void); PyAPI_FUNC(PyObject *) PyDict_GetItem(PyObject *mp, PyObject *key); diff --git a/Include/floatobject.h b/Include/floatobject.h index 0fb9fc4e0fae7..917dfcc26445c 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -21,7 +21,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyFloat_Type; #define PyFloat_Check(op) PyObject_TypeCheck(op, &PyFloat_Type) -#define PyFloat_CheckExact(op) (Py_TYPE(op) == &PyFloat_Type) +#define PyFloat_CheckExact(op) Py_IS_TYPE(op, &PyFloat_Type) #ifdef Py_NAN #define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) diff --git a/Include/funcobject.h b/Include/funcobject.h index c6dd67d6124d3..c5cc9d261a314 100644 --- a/Include/funcobject.h +++ b/Include/funcobject.h @@ -43,7 +43,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyFunction_Type; -#define PyFunction_Check(op) (Py_TYPE(op) == &PyFunction_Type) +#define PyFunction_Check(op) Py_IS_TYPE(op, &PyFunction_Type) PyAPI_FUNC(PyObject *) PyFunction_New(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) PyFunction_NewWithQualName(PyObject *, PyObject *, PyObject *); diff --git a/Include/genobject.h b/Include/genobject.h index 5ee9a2831d12b..b87a6485631c0 100644 --- a/Include/genobject.h +++ b/Include/genobject.h @@ -38,7 +38,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyGen_Type; #define PyGen_Check(op) PyObject_TypeCheck(op, &PyGen_Type) -#define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type) +#define PyGen_CheckExact(op) Py_IS_TYPE(op, &PyGen_Type) PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *); PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(struct _frame *, @@ -58,7 +58,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_DATA(PyTypeObject) _PyCoroWrapper_Type; -#define PyCoro_CheckExact(op) (Py_TYPE(op) == &PyCoro_Type) +#define PyCoro_CheckExact(op) Py_IS_TYPE(op, &PyCoro_Type) PyObject *_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *) PyCoro_New(struct _frame *, PyObject *name, PyObject *qualname); @@ -89,7 +89,7 @@ PyAPI_DATA(PyTypeObject) _PyAsyncGenAThrow_Type; PyAPI_FUNC(PyObject *) PyAsyncGen_New(struct _frame *, PyObject *name, PyObject *qualname); -#define PyAsyncGen_CheckExact(op) (Py_TYPE(op) == &PyAsyncGen_Type) +#define PyAsyncGen_CheckExact(op) Py_IS_TYPE(op, &PyAsyncGen_Type) PyObject *_PyAsyncGenValueWrapperNew(PyObject *); diff --git a/Include/internal/pycore_hamt.h b/Include/internal/pycore_hamt.h index e65aef5e21a95..aaf655909551a 100644 --- a/Include/internal/pycore_hamt.h +++ b/Include/internal/pycore_hamt.h @@ -8,7 +8,7 @@ #define _Py_HAMT_MAX_TREE_DEPTH 7 -#define PyHamt_Check(o) (Py_TYPE(o) == &_PyHamt_Type) +#define PyHamt_Check(o) Py_IS_TYPE(o, &_PyHamt_Type) /* Abstract tree node. */ diff --git a/Include/iterobject.h b/Include/iterobject.h index eec2ee271eb67..51139bf187408 100644 --- a/Include/iterobject.h +++ b/Include/iterobject.h @@ -8,12 +8,12 @@ extern "C" { PyAPI_DATA(PyTypeObject) PySeqIter_Type; PyAPI_DATA(PyTypeObject) PyCallIter_Type; -#define PySeqIter_Check(op) (Py_TYPE(op) == &PySeqIter_Type) +#define PySeqIter_Check(op) Py_IS_TYPE(op, &PySeqIter_Type) PyAPI_FUNC(PyObject *) PySeqIter_New(PyObject *); -#define PyCallIter_Check(op) (Py_TYPE(op) == &PyCallIter_Type) +#define PyCallIter_Check(op) Py_IS_TYPE(op, &PyCallIter_Type) PyAPI_FUNC(PyObject *) PyCallIter_New(PyObject *, PyObject *); diff --git a/Include/listobject.h b/Include/listobject.h index 34dfcf92ec93a..2a8a25525d1d7 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -23,7 +23,7 @@ PyAPI_DATA(PyTypeObject) PyListRevIter_Type; #define PyList_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_LIST_SUBCLASS) -#define PyList_CheckExact(op) (Py_TYPE(op) == &PyList_Type) +#define PyList_CheckExact(op) Py_IS_TYPE(op, &PyList_Type) PyAPI_FUNC(PyObject *) PyList_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyList_Size(PyObject *); diff --git a/Include/memoryobject.h b/Include/memoryobject.h index 990a716f22039..306028f4b225d 100644 --- a/Include/memoryobject.h +++ b/Include/memoryobject.h @@ -11,7 +11,7 @@ PyAPI_DATA(PyTypeObject) _PyManagedBuffer_Type; #endif PyAPI_DATA(PyTypeObject) PyMemoryView_Type; -#define PyMemoryView_Check(op) (Py_TYPE(op) == &PyMemoryView_Type) +#define PyMemoryView_Check(op) Py_IS_TYPE(op, &PyMemoryView_Type) #ifndef Py_LIMITED_API /* Get a pointer to the memoryview's private copy of the exporter's buffer. */ diff --git a/Include/methodobject.h b/Include/methodobject.h index d9f8d4f80c2cd..adb2d9e884fbb 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -13,7 +13,7 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyCFunction_Type; -#define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type) +#define PyCFunction_Check(op) Py_IS_TYPE(op, &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); typedef PyObject *(*_PyCFunctionFast) (PyObject *, PyObject *const *, Py_ssize_t); diff --git a/Include/moduleobject.h b/Include/moduleobject.h index e246fd2faf918..cf9ad40c0a17a 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -10,7 +10,7 @@ extern "C" { PyAPI_DATA(PyTypeObject) PyModule_Type; #define PyModule_Check(op) PyObject_TypeCheck(op, &PyModule_Type) -#define PyModule_CheckExact(op) (Py_TYPE(op) == &PyModule_Type) +#define PyModule_CheckExact(op) Py_IS_TYPE(op, &PyModule_Type) #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 PyAPI_FUNC(PyObject *) PyModule_NewObject( diff --git a/Include/object.h b/Include/object.h index 11539ee080503..3d0da52c2b6b1 100644 --- a/Include/object.h +++ b/Include/object.h @@ -123,6 +123,11 @@ typedef struct { #define Py_TYPE(ob) (_PyObject_CAST(ob)->ob_type) #define Py_SIZE(ob) (_PyVarObject_CAST(ob)->ob_size) +static inline int _Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { + return ob->ob_type == type; +} +#define Py_IS_TYPE(ob, type) _Py_IS_TYPE(_PyObject_CAST(ob), type) + static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { ob->ob_refcnt = refcnt; } @@ -211,7 +216,7 @@ PyAPI_FUNC(void*) PyType_GetSlot(PyTypeObject*, int); /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); #define PyObject_TypeCheck(ob, tp) \ - (Py_TYPE(ob) == (tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) + (Py_IS_TYPE(ob, tp) || PyType_IsSubtype(Py_TYPE(ob), (tp))) PyAPI_DATA(PyTypeObject) PyType_Type; /* built-in 'type' */ PyAPI_DATA(PyTypeObject) PyBaseObject_Type; /* built-in 'object' */ @@ -623,7 +628,7 @@ static inline int _PyType_Check(PyObject *op) { #define PyType_Check(op) _PyType_Check(_PyObject_CAST(op)) static inline int _PyType_CheckExact(PyObject *op) { - return (Py_TYPE(op) == &PyType_Type); + return Py_IS_TYPE(op, &PyType_Type); } #define PyType_CheckExact(op) _PyType_CheckExact(_PyObject_CAST(op)) diff --git a/Include/odictobject.h b/Include/odictobject.h index 35aff8a29a6e3..e070413017d80 100644 --- a/Include/odictobject.h +++ b/Include/odictobject.h @@ -19,7 +19,7 @@ PyAPI_DATA(PyTypeObject) PyODictItems_Type; PyAPI_DATA(PyTypeObject) PyODictValues_Type; #define PyODict_Check(op) PyObject_TypeCheck(op, &PyODict_Type) -#define PyODict_CheckExact(op) (Py_TYPE(op) == &PyODict_Type) +#define PyODict_CheckExact(op) Py_IS_TYPE(op, &PyODict_Type) #define PyODict_SIZE(op) PyDict_GET_SIZE((op)) PyAPI_FUNC(PyObject *) PyODict_New(void); diff --git a/Include/pycapsule.h b/Include/pycapsule.h index d9ecda7a4b6e4..fb5d503fea73f 100644 --- a/Include/pycapsule.h +++ b/Include/pycapsule.h @@ -22,7 +22,7 @@ PyAPI_DATA(PyTypeObject) PyCapsule_Type; typedef void (*PyCapsule_Destructor)(PyObject *); -#define PyCapsule_CheckExact(op) (Py_TYPE(op) == &PyCapsule_Type) +#define PyCapsule_CheckExact(op) Py_IS_TYPE(op, &PyCapsule_Type) PyAPI_FUNC(PyObject *) PyCapsule_New( diff --git a/Include/rangeobject.h b/Include/rangeobject.h index 7e4dc28894b04..d6af8473f9e8d 100644 --- a/Include/rangeobject.h +++ b/Include/rangeobject.h @@ -19,7 +19,7 @@ PyAPI_DATA(PyTypeObject) PyRange_Type; PyAPI_DATA(PyTypeObject) PyRangeIter_Type; PyAPI_DATA(PyTypeObject) PyLongRangeIter_Type; -#define PyRange_Check(op) (Py_TYPE(op) == &PyRange_Type) +#define PyRange_Check(op) Py_IS_TYPE(op, &PyRange_Type) #ifdef __cplusplus } diff --git a/Include/setobject.h b/Include/setobject.h index fc0ea83925f92..05a097eba7f7d 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -88,18 +88,18 @@ PyAPI_FUNC(int) PySet_Discard(PyObject *set, PyObject *key); PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); PyAPI_FUNC(Py_ssize_t) PySet_Size(PyObject *anyset); -#define PyFrozenSet_CheckExact(ob) (Py_TYPE(ob) == &PyFrozenSet_Type) +#define PyFrozenSet_CheckExact(ob) Py_IS_TYPE(ob, &PyFrozenSet_Type) #define PyAnySet_CheckExact(ob) \ - (Py_TYPE(ob) == &PySet_Type || Py_TYPE(ob) == &PyFrozenSet_Type) + (Py_IS_TYPE(ob, &PySet_Type) || Py_IS_TYPE(ob, &PyFrozenSet_Type)) #define PyAnySet_Check(ob) \ - (Py_TYPE(ob) == &PySet_Type || Py_TYPE(ob) == &PyFrozenSet_Type || \ + (Py_IS_TYPE(ob, &PySet_Type) || Py_IS_TYPE(ob, &PyFrozenSet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) #define PySet_Check(ob) \ - (Py_TYPE(ob) == &PySet_Type || \ + (Py_IS_TYPE(ob, &PySet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PySet_Type)) #define PyFrozenSet_Check(ob) \ - (Py_TYPE(ob) == &PyFrozenSet_Type || \ + (Py_IS_TYPE(ob, &PyFrozenSet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) #ifdef __cplusplus diff --git a/Include/sliceobject.h b/Include/sliceobject.h index aae6f3cc7945e..2c889508b4b49 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -28,7 +28,7 @@ typedef struct { PyAPI_DATA(PyTypeObject) PySlice_Type; PyAPI_DATA(PyTypeObject) PyEllipsis_Type; -#define PySlice_Check(op) (Py_TYPE(op) == &PySlice_Type) +#define PySlice_Check(op) Py_IS_TYPE(op, &PySlice_Type) PyAPI_FUNC(PyObject *) PySlice_New(PyObject* start, PyObject* stop, PyObject* step); diff --git a/Include/symtable.h b/Include/symtable.h index 5dcfa7e2c2bb6..abd19a7923e1b 100644 --- a/Include/symtable.h +++ b/Include/symtable.h @@ -69,7 +69,7 @@ typedef struct _symtable_entry { PyAPI_DATA(PyTypeObject) PySTEntry_Type; -#define PySTEntry_Check(op) (Py_TYPE(op) == &PySTEntry_Type) +#define PySTEntry_Check(op) Py_IS_TYPE(op, &PySTEntry_Type) PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *); diff --git a/Include/traceback.h b/Include/traceback.h index b451927fafa3a..0efbae8a76a2f 100644 --- a/Include/traceback.h +++ b/Include/traceback.h @@ -13,7 +13,7 @@ PyAPI_FUNC(int) PyTraceBack_Print(PyObject *, PyObject *); /* Reveal traceback type so we can typecheck traceback objects */ PyAPI_DATA(PyTypeObject) PyTraceBack_Type; -#define PyTraceBack_Check(v) (Py_TYPE(v) == &PyTraceBack_Type) +#define PyTraceBack_Check(v) Py_IS_TYPE(v, &PyTraceBack_Type) #ifndef Py_LIMITED_API diff --git a/Include/tupleobject.h b/Include/tupleobject.h index 590902de9d021..d3504b0501f9e 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -25,7 +25,7 @@ PyAPI_DATA(PyTypeObject) PyTupleIter_Type; #define PyTuple_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TUPLE_SUBCLASS) -#define PyTuple_CheckExact(op) (Py_TYPE(op) == &PyTuple_Type) +#define PyTuple_CheckExact(op) Py_IS_TYPE(op, &PyTuple_Type) PyAPI_FUNC(PyObject *) PyTuple_New(Py_ssize_t size); PyAPI_FUNC(Py_ssize_t) PyTuple_Size(PyObject *); diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 4dea494218122..500ce242e9f0e 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -113,7 +113,7 @@ PyAPI_DATA(PyTypeObject) PyUnicodeIter_Type; #define PyUnicode_Check(op) \ PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_UNICODE_SUBCLASS) -#define PyUnicode_CheckExact(op) (Py_TYPE(op) == &PyUnicode_Type) +#define PyUnicode_CheckExact(op) Py_IS_TYPE(op, &PyUnicode_Type) /* --- Constants ---------------------------------------------------------- */ diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index 17051568f3a6e..ac4b4821c8a14 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -46,10 +46,10 @@ PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; #define PyWeakref_CheckRef(op) PyObject_TypeCheck(op, &_PyWeakref_RefType) #define PyWeakref_CheckRefExact(op) \ - (Py_TYPE(op) == &_PyWeakref_RefType) + Py_IS_TYPE(op, &_PyWeakref_RefType) #define PyWeakref_CheckProxy(op) \ - ((Py_TYPE(op) == &_PyWeakref_ProxyType) || \ - (Py_TYPE(op) == &_PyWeakref_CallableProxyType)) + (Py_IS_TYPE(op, &_PyWeakref_ProxyType) || \ + Py_IS_TYPE(op, &_PyWeakref_CallableProxyType)) #define PyWeakref_Check(op) \ (PyWeakref_CheckRef(op) || PyWeakref_CheckProxy(op)) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-13-01-30-22.bpo-39573.uTFj1m.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-13-01-30-22.bpo-39573.uTFj1m.rst new file mode 100644 index 0000000000000..56e7e1ba3242c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-13-01-30-22.bpo-39573.uTFj1m.rst @@ -0,0 +1,2 @@ +Add :c:func:`Py_IS_TYPE` static inline function to check +whether the object *o* type is *type*. diff --git a/Objects/genobject.c b/Objects/genobject.c index 0efd57de7a5a3..0837698fd784c 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -255,7 +255,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) if (PyCoro_CheckExact(gen)) { msg = "coroutine raised StopIteration"; } - else if PyAsyncGen_CheckExact(gen) { + else if (PyAsyncGen_CheckExact(gen)) { msg = "async generator raised StopIteration"; } _PyErr_FormatFromCause(PyExc_RuntimeError, "%s", msg);

1 0

bpo-39573: Fix bad copy-paste in Py_SET_SIZE (GH-18496)
by Brandt Bucher 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/968dcd9e7a4d3aa9aaa1dfca693adf60d6… commit: 968dcd9e7a4d3aa9aaa1dfca693adf60d6b71ce7 branch: master author: Brandt Bucher <brandtbucher(a)gmail.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T18:34:45+01:00 summary: bpo-39573: Fix bad copy-paste in Py_SET_SIZE (GH-18496) files: M Doc/c-api/structures.rst M Include/object.h diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 75e2383beb216..083c3740531e4 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -103,7 +103,7 @@ the definition of all other Python objects. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - Set the object *o* size of *size*. + Set the object *o* size to *size*. .. versionadded:: 3.9 diff --git a/Include/object.h b/Include/object.h index 68200f7666f17..11539ee080503 100644 --- a/Include/object.h +++ b/Include/object.h @@ -133,10 +133,10 @@ static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { } #define Py_SET_TYPE(ob, type) _Py_SET_TYPE(_PyObject_CAST(ob), type) -static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t refcnt) { - ob->ob_size = refcnt; +static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { + ob->ob_size = size; } -#define Py_SET_SIZE(ob, refcnt) _Py_SET_SIZE(_PyVarObject_CAST(ob), refcnt) +#define Py_SET_SIZE(ob, size) _Py_SET_SIZE(_PyVarObject_CAST(ob), size) /*

1 0

bpo-39606: allow closing async generators that are already closed (GH-18475) (GH-18502)
by Nathaniel J. Smith 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/f464edf3239f7867fe31c9cd238a68fb3b… commit: f464edf3239f7867fe31c9cd238a68fb3b90feaa branch: 3.7 author: Nathaniel J. Smith <njs(a)pobox.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T01:33:35-08:00 summary: bpo-39606: allow closing async generators that are already closed (GH-18475) (GH-18502) The fix for [bpo-39386](https://bugs.python.org/issue39386) attempted to make it so you couldn't reuse a agen.aclose() coroutine object. It accidentally also prevented you from calling aclose() at all on an async generator that was already closed or exhausted. This commit fixes it so we're only blocking the actually illegal cases, while allowing the legal cases. The new tests failed before this patch. Also confirmed that this fixes the test failures we were seeing in Trio with Python dev builds: https://github.com/python-trio/trio/pull/1396 https://bugs.python.org/issue39606 (cherry picked from commit 925dc7fb1d0db85dc137afa4cd14211bf0d67414) files: A Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst M Lib/test/test_asyncgen.py M Objects/genobject.c diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 426d5d1161423..5a292fb5d64e1 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1117,7 +1117,7 @@ def exception_handler(loop, context): self.assertEqual([], messages) - def test_async_gen_await_anext_twice(self): + def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): yield 1 yield 2 @@ -1136,7 +1136,7 @@ def test_async_gen_await_anext_twice(self): self.loop.run_until_complete(run()) - def test_async_gen_await_aclose_twice(self): + def test_async_gen_await_same_aclose_coro_twice(self): async def async_iterate(): yield 1 yield 2 @@ -1153,6 +1153,32 @@ def test_async_gen_await_aclose_twice(self): self.loop.run_until_complete(run()) + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: + pass + await it.aclose() + + self.loop.run_until_complete(run()) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst new file mode 100644 index 0000000000000..b7cbe4e91f59c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst @@ -0,0 +1,2 @@ +Fix regression caused by fix for bpo-39386, that prevented calling +``aclose`` on an async generator that had already been closed or exhausted. diff --git a/Objects/genobject.c b/Objects/genobject.c index a42169f82e4d7..dd7d44bd427df 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1804,14 +1804,19 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) PyFrameObject *f = gen->gi_frame; PyObject *retval; - if (f == NULL || f->f_stacktop == NULL || - o->agt_state == AWAITABLE_STATE_CLOSED) { + if (o->agt_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( PyExc_RuntimeError, "cannot reuse already awaited aclose()/athrow()"); return NULL; } + if (f == NULL || f->f_stacktop == NULL) { + o->agt_state = AWAITABLE_STATE_CLOSED; + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_closed) { PyErr_SetNone(PyExc_StopIteration); @@ -1882,6 +1887,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) } yield_close: + o->agt_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; @@ -1924,6 +1930,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args) } else { /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { + o->agt_state = AWAITABLE_STATE_CLOSED; Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL;

1 0

[3.8] bpo-39606: allow closing async generators that are already closed (GH-18475) (GH-18501)
by Miss Islington (bot) 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/8dbdf5f275c6462bb522bcf3a29054239d… commit: 8dbdf5f275c6462bb522bcf3a29054239d72989d branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T00:43:23-08:00 summary: [3.8] bpo-39606: allow closing async generators that are already closed (GH-18475) (GH-18501) The fix for [bpo-39386](https://bugs.python.org/issue39386) attempted to make it so you couldn't reuse a agen.aclose() coroutine object. It accidentally also prevented you from calling aclose() at all on an async generator that was already closed or exhausted. This commit fixes it so we're only blocking the actually illegal cases, while allowing the legal cases. The new tests failed before this patch. Also confirmed that this fixes the test failures we were seeing in Trio with Python dev builds: https://github.com/python-trio/trio/pull/1396 https://bugs.python.org/issue39606 (cherry picked from commit 925dc7fb1d0db85dc137afa4cd14211bf0d67414) Co-authored-by: Nathaniel J. Smith <njs(a)pobox.com> https://bugs.python.org/issue39606 Automerge-Triggered-By: @njsmith files: A Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst M Lib/test/test_asyncgen.py M Objects/genobject.c diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 24b20bec2b2d1..fb6321d2264f3 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1128,7 +1128,7 @@ def exception_handler(loop, context): self.assertEqual([], messages) - def test_async_gen_await_anext_twice(self): + def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): yield 1 yield 2 @@ -1147,7 +1147,7 @@ def test_async_gen_await_anext_twice(self): self.loop.run_until_complete(run()) - def test_async_gen_await_aclose_twice(self): + def test_async_gen_await_same_aclose_coro_twice(self): async def async_iterate(): yield 1 yield 2 @@ -1164,6 +1164,32 @@ def test_async_gen_await_aclose_twice(self): self.loop.run_until_complete(run()) + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: + pass + await it.aclose() + + self.loop.run_until_complete(run()) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst new file mode 100644 index 0000000000000..b7cbe4e91f59c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst @@ -0,0 +1,2 @@ +Fix regression caused by fix for bpo-39386, that prevented calling +``aclose`` on an async generator that had already been closed or exhausted. diff --git a/Objects/genobject.c b/Objects/genobject.c index 72aa872c6b59f..2c06bdcc72642 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1812,16 +1812,22 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) PyFrameObject *f = gen->gi_frame; PyObject *retval; - if (f == NULL || f->f_stacktop == NULL || - o->agt_state == AWAITABLE_STATE_CLOSED) { + if (o->agt_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( PyExc_RuntimeError, "cannot reuse already awaited aclose()/athrow()"); return NULL; } + if (f == NULL || f->f_stacktop == NULL) { + o->agt_state = AWAITABLE_STATE_CLOSED; + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; if (o->agt_args == NULL) { PyErr_SetString( PyExc_RuntimeError, @@ -1893,7 +1899,6 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) /* aclose() mode */ if (retval) { if (_PyAsyncGenWrappedValue_CheckExact(retval)) { - o->agt_gen->ag_running_async = 0; Py_DECREF(retval); goto yield_close; } @@ -1908,16 +1913,17 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) yield_close: o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; check_error: o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - o->agt_state = AWAITABLE_STATE_CLOSED; if (o->agt_args == NULL) { /* when aclose() is called we don't want to propagate StopAsyncIteration or GeneratorExit; just raise @@ -1951,6 +1957,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args) /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL;

1 0

bpo-39184: Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, `shutil`, `signal`, `syslog` (GH-18407)
by Steve Dower 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/a00b5be5f71b702ab80b0a7c046485046a… commit: a00b5be5f71b702ab80b0a7c046485046aaae160 branch: 3.8 author: Steve Dower <steve.dower(a)python.org> committer: GitHub <noreply(a)github.com> date: 2020-02-13T08:30:27Z summary: bpo-39184: Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, `shutil`, `signal`, `syslog` (GH-18407) Co-authored-by: Saiyang Gou <gousaiyang(a)163.com> files: A Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst M Doc/library/fcntl.rst M Doc/library/msvcrt.rst M Doc/library/os.rst M Doc/library/resource.rst M Doc/library/shutil.rst M Doc/library/signal.rst M Doc/library/syslog.rst M Lib/shutil.py M Modules/fcntlmodule.c M Modules/posixmodule.c M Modules/resource.c M Modules/signalmodule.c M Modules/syslogmodule.c M PC/msvcrtmodule.c diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index a7390150f7910..69484b647363d 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -57,6 +57,8 @@ The module defines the following functions: If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl + .. function:: ioctl(fd, request, arg=0, mutate_flag=True) @@ -106,6 +108,8 @@ The module defines the following functions: >>> buf array('h', [13341]) + .. audit-event:: fcntl.ioctl fd,request,arg fcntl.ioctl + .. function:: flock(fd, operation) @@ -116,6 +120,8 @@ The module defines the following functions: If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + .. audit-event:: fcntl.flock fd,operation fcntl.flock + .. function:: lockf(fd, cmd, len=0, start=0, whence=0) @@ -149,6 +155,8 @@ The module defines the following functions: The default for *len* is 0 which means to lock to the end of the file. The default for *whence* is also 0. + .. audit-event:: fcntl.lockf fd,cmd,len,start,whence fcntl.lockf + Examples (all on a SVR4 compliant system):: import struct, fcntl, os diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 14ad2cd4373af..42fffee6a0f44 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -42,6 +42,8 @@ File Operations regions in a file may be locked at the same time, but may not overlap. Adjacent regions are not merged; they must be unlocked individually. + .. audit-event:: msvcrt.locking fd,mode,nbytes msvcrt.locking + .. data:: LK_LOCK LK_RLCK @@ -77,12 +79,16 @@ File Operations and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. + .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle + .. function:: get_osfhandle(fd) Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if *fd* is not recognized. + .. audit-event:: msvcrt.get_osfhandle fd msvcrt.get_osfhandle + .. _msvcrt-console: diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 0d8df34c345c4..3157b887b535c 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -451,6 +451,8 @@ process and user. calls to :func:`putenv` don't update ``os.environ``, so it is actually preferable to assign to items of ``os.environ``. + .. audit-event:: os.putenv key,value os.putenv + .. function:: setegid(egid) @@ -643,6 +645,8 @@ process and user. calls to :func:`unsetenv` don't update ``os.environ``, so it is actually preferable to delete items of ``os.environ``. + .. audit-event:: os.unsetenv key os.unsetenv + .. availability:: most flavors of Unix. @@ -768,6 +772,8 @@ as internal buffering of data. docs for :func:`chmod` for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(fd, mode)``. + .. audit-event:: os.chmod path,mode,dir_fd os.fchmod + .. availability:: Unix. @@ -778,6 +784,8 @@ as internal buffering of data. :func:`chown`. As of Python 3.3, this is equivalent to ``os.chown(fd, uid, gid)``. + .. audit-event:: os.chown path,uid,gid,dir_fd os.fchown + .. availability:: Unix. @@ -885,6 +893,8 @@ as internal buffering of data. :data:`F_ULOCK` or :data:`F_TEST`. *len* specifies the section of the file to lock. + .. audit-event:: os.lockf fd,cmd,len os.lockf + .. availability:: Unix. .. versionadded:: 3.3 @@ -1602,6 +1612,8 @@ features: This function can raise :exc:`OSError` and subclasses such as :exc:`FileNotFoundError`, :exc:`PermissionError`, and :exc:`NotADirectoryError`. + .. audit-event:: os.chdir path os.chdir + .. versionadded:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -1630,6 +1642,8 @@ features: This function can support :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.chflags path,flags os.chflags + .. availability:: Unix. .. versionadded:: 3.3 @@ -1675,6 +1689,8 @@ features: read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` constants or a corresponding integer value). All other bits are ignored. + .. audit-event:: os.chmod path,mode,dir_fd os.chmod + .. versionadded:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -1695,6 +1711,8 @@ features: See :func:`shutil.chown` for a higher-level function that accepts names in addition to numeric ids. + .. audit-event:: os.chown path,uid,gid,dir_fd os.chown + .. availability:: Unix. .. versionadded:: 3.3 @@ -1721,6 +1739,8 @@ features: descriptor *fd*. The descriptor must refer to an opened directory, not an open file. As of Python 3.3, this is equivalent to ``os.chdir(fd)``. + .. audit-event:: os.chdir path os.fchdir + .. availability:: Unix. @@ -1745,6 +1765,8 @@ features: not follow symbolic links. As of Python 3.3, this is equivalent to ``os.chflags(path, flags, follow_symlinks=False)``. + .. audit-event:: os.chflags path,flags os.lchflags + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1758,6 +1780,8 @@ features: for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(path, mode, follow_symlinks=False)``. + .. audit-event:: os.chmod path,mode,dir_fd os.lchmod + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1769,6 +1793,8 @@ features: function will not follow symbolic links. As of Python 3.3, this is equivalent to ``os.chown(path, uid, gid, follow_symlinks=False)``. + .. audit-event:: os.chown path,uid,gid,dir_fd os.lchown + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1783,6 +1809,8 @@ features: supply :ref:`paths relative to directory descriptors <dir_fd>`, and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -1885,6 +1913,8 @@ features: It is also possible to create temporary directories; see the :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. + .. audit-event:: os.mkdir path,mode,dir_fd os.mkdir + .. versionadded:: 3.3 The *dir_fd* argument. @@ -1917,6 +1947,8 @@ features: This function handles UNC paths correctly. + .. audit-event:: os.mkdir path,mode,dir_fd os.makedirs + .. versionadded:: 3.2 The *exist_ok* parameter. @@ -2082,6 +2114,8 @@ features: This function is semantically identical to :func:`unlink`. + .. audit-event:: os.remove path,dir_fd os.remove + .. versionadded:: 3.3 The *dir_fd* argument. @@ -2102,6 +2136,8 @@ features: they are empty. Raises :exc:`OSError` if the leaf directory could not be successfully removed. + .. audit-event:: os.remove path,dir_fd os.removedirs + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2127,6 +2163,8 @@ features: If you want cross-platform overwriting of the destination, use :func:`replace`. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.rename + .. versionadded:: 3.3 The *src_dir_fd* and *dst_dir_fd* arguments. @@ -2146,6 +2184,8 @@ features: This function can fail with the new directory structure made if you lack permissions needed to remove the leaf directory or file. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.renames + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *old* and *new*. @@ -2161,6 +2201,8 @@ features: This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to supply :ref:`paths relative to directory descriptors <dir_fd>`. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.replace + .. versionadded:: 3.3 .. versionchanged:: 3.6 @@ -2177,6 +2219,8 @@ features: This function can support :ref:`paths relative to directory descriptors <dir_fd>`. + .. audit-event:: os.rmdir path,dir_fd os.rmdir + .. versionadded:: 3.3 The *dir_fd* parameter. @@ -2820,6 +2864,8 @@ features: :exc:`OSError` is raised when the function is called by an unprivileged user. + .. audit-event:: os.symlink src,dst,dir_fd os.symlink + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -2872,6 +2918,8 @@ features: traditional Unix name. Please see the documentation for :func:`remove` for further information. + .. audit-event:: os.remove path,dir_fd os.unlink + .. versionadded:: 3.3 The *dir_fd* parameter. @@ -2909,6 +2957,8 @@ features: :ref:`paths relative to directory descriptors <dir_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.utime path,times,ns,dir_fd os.utime + .. versionadded:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd*, *follow_symlinks*, and *ns* parameters. @@ -3134,6 +3184,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor <path_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.getxattr path,attribute os.getxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3148,6 +3200,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor <path_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.listxattr path os.listxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3162,6 +3216,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor <path_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.removexattr path,attribute os.removexattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3185,6 +3241,8 @@ These functions are all available on Linux only. A bug in Linux kernel versions less than 2.6.39 caused the flags argument to be ignored on some filesystems. + .. audit-event:: os.setxattr path,attribute,value,flags os.setxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3247,6 +3305,8 @@ to be ignored. <https://msdn.microsoft.com/44228cf2-6306-466c-8f16-f513cd3ba8b5>`_ for more information about how DLLs are loaded. + .. audit-event:: os.add_dll_directory path os.add_dll_directory + .. availability:: Windows. .. versionadded:: 3.8 @@ -3479,6 +3539,8 @@ written in Python, such as a mail server's external command delivery program. Note that some platforms including FreeBSD <= 6.3 and Cygwin have known issues when using ``fork()`` from a thread. + .. audit-event:: os.fork "" os.fork + .. versionchanged:: 3.8 Calling ``fork()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -3498,6 +3560,8 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + .. audit-event:: os.forkpty "" os.forkpty + .. versionchanged:: 3.8 Calling ``forkpty()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -3524,6 +3588,8 @@ written in Python, such as a mail server's external command delivery program. See also :func:`signal.pthread_kill`. + .. audit-event:: os.kill pid,sig os.kill + .. versionadded:: 3.2 Windows support. @@ -3536,6 +3602,8 @@ written in Python, such as a mail server's external command delivery program. Send the signal *sig* to the process group *pgid*. + .. audit-event:: os.killpg pgid,sig os.killpg + .. availability:: Unix. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 3573da7ea2d71..e4eac43642d14 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -78,6 +78,9 @@ this module for those platforms. VxWorks only supports setting :data:`RLIMIT_NOFILE`. + .. audit-event:: resource.setrlimit resource,limits resource.setrlimit + + .. function:: prlimit(pid, resource[, limits]) Combines :func:`setrlimit` and :func:`getrlimit` in one function and @@ -94,6 +97,8 @@ this module for those platforms. :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for the process. + .. audit-event:: resource.prlimit pid,resource,limits resource.prlimit + .. availability:: Linux 2.6.36 or later with glibc 2.13 or later. .. versionadded:: 3.4 diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 174b7e875a38a..bd24de7202321 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -67,6 +67,8 @@ Directory and files operations a new symbolic link will be created instead of copying the file *src* points to. + .. audit-event:: shutil.copyfile src,dst shutil.copyfile + .. versionchanged:: 3.3 :exc:`IOError` used to be raised instead of :exc:`OSError`. Added *follow_symlinks* argument. @@ -101,6 +103,8 @@ Directory and files operations :func:`copymode` cannot modify symbolic links on the local platform, and it is asked to do so, it will do nothing and return. + .. audit-event:: shutil.copymode src,dst shutil.copymode + .. versionchanged:: 3.3 Added *follow_symlinks* argument. @@ -146,6 +150,8 @@ Directory and files operations Please see :data:`os.supports_follow_symlinks` for more information. + .. audit-event:: shutil.copystat src,dst shutil.copystat + .. versionchanged:: 3.3 Added *follow_symlinks* argument and support for Linux extended attributes. @@ -167,6 +173,10 @@ Directory and files operations To preserve all file metadata from the original, use :func:`~shutil.copy2` instead. + .. audit-event:: shutil.copyfile src,dst shutil.copy + + .. audit-event:: shutil.copymode src,dst shutil.copy + .. versionchanged:: 3.3 Added *follow_symlinks* argument. Now returns path to the newly created file. @@ -194,6 +204,10 @@ Directory and files operations Please see :func:`copystat` for more information about platform support for modifying symbolic link metadata. + .. audit-event:: shutil.copyfile src,dst shutil.copy2 + + .. audit-event:: shutil.copystat src,dst shutil.copy2 + .. versionchanged:: 3.3 Added *follow_symlinks* argument, try to copy extended file system attributes too (currently Linux only). @@ -342,6 +356,8 @@ Directory and files operations *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. + .. audit-event:: shutil.move src,dst shutil.move + .. versionchanged:: 3.3 Added explicit symlink handling for foreign filesystems, thus adapting it to the behavior of GNU's :program:`mv`. @@ -378,6 +394,8 @@ Directory and files operations See also :func:`os.chown`, the underlying function. + .. audit-event:: shutil.chown path,user,group shutil.chown + .. availability:: Unix. .. versionadded:: 3.3 @@ -629,6 +647,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. registered for that extension. In case none is found, a :exc:`ValueError` is raised. + .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive + .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 8fecc2b7eed0e..932201b0e9d7d 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -264,6 +264,8 @@ The :mod:`signal` module defines the following functions: If *signalnum* is 0, then no signal is sent, but error checking is still performed; this can be used to check if the target thread is still running. + .. audit-event:: signal.pthread_kill thread_id,signalnum signal.pthread_kill + .. availability:: Unix. See the man page :manpage:`pthread_kill(3)` for further information. diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 7151527ce57a5..d264a3340c98b 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -31,6 +31,8 @@ The module defines the following functions: If :func:`openlog` has not been called prior to the call to :func:`syslog`, ``openlog()`` will be called with no arguments. + .. audit-event:: syslog.syslog priority,message syslog.syslog + .. function:: openlog([ident[, logoption[, facility]]]) @@ -45,6 +47,8 @@ The module defines the following functions: keyword argument (default is :const:`LOG_USER`) sets the default facility for messages which do not have a facility explicitly encoded. + .. audit-event:: syslog.openlog ident,logoption,facility syslog.openlog + .. versionchanged:: 3.2 In previous versions, keyword arguments were not allowed, and *ident* was required. The default for *ident* was dependent on the system libraries, @@ -60,6 +64,8 @@ The module defines the following functions: :func:`openlog` hasn't already been called), and *ident* and other :func:`openlog` parameters are reset to defaults. + .. audit-event:: syslog.closelog "" syslog.closelog + .. function:: setlogmask(maskpri) @@ -70,6 +76,8 @@ The module defines the following functions: ``LOG_UPTO(pri)`` calculates the mask for all priorities up to and including *pri*. + .. audit-event:: syslog.setlogmask maskpri syslog.setlogmask + The module defines the following constants: Priority levels (high to low): diff --git a/Lib/shutil.py b/Lib/shutil.py index cde7b860050a2..1f05d80f32a8f 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -235,6 +235,8 @@ def copyfile(src, dst, *, follow_symlinks=True): symlink will be created instead of copying the file it points to. """ + sys.audit("shutil.copyfile", src, dst) + if _samefile(src, dst): raise SameFileError("{!r} and {!r} are the same file".format(src, dst)) @@ -289,6 +291,8 @@ def copymode(src, dst, *, follow_symlinks=True): (e.g. Linux) this method does nothing. """ + sys.audit("shutil.copymode", src, dst) + if not follow_symlinks and _islink(src) and os.path.islink(dst): if hasattr(os, 'lchmod'): stat_func, chmod_func = os.lstat, os.lchmod @@ -340,6 +344,8 @@ def copystat(src, dst, *, follow_symlinks=True): If the optional flag `follow_symlinks` is not set, symlinks aren't followed if and only if both `src` and `dst` are symlinks. """ + sys.audit("shutil.copystat", src, dst) + def _nop(*args, ns=None, follow_symlinks=None): pass @@ -766,6 +772,7 @@ def move(src, dst, copy_function=copy2): the issues this implementation glosses over. """ + sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): if _samefile(src, dst): @@ -1193,6 +1200,8 @@ def unpack_archive(filename, extract_dir=None, format=None): In case none is found, a ValueError is raised. """ + sys.audit("shutil.unpack_archive", filename, extract_dir, format) + if extract_dir is None: extract_dir = os.getcwd() @@ -1260,6 +1269,7 @@ def chown(path, user=None, group=None): user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. """ + sys.audit('shutil.chown', path, user, group) if user is None and group is None: raise ValueError("user and/or group must be set") diff --git a/Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst b/Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst new file mode 100644 index 0000000000000..cf25c24d58788 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst @@ -0,0 +1 @@ +Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, `shutil`, `signal` and `syslog`. \ No newline at end of file diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 0fbf7876c3e20..a7d2193022e98 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -66,6 +66,10 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) char buf[1024]; int async_err = 0; + if (PySys_Audit("fcntl.fcntl", "iiO", fd, code, arg ? arg : Py_None) < 0) { + return NULL; + } + if (arg != NULL) { int parse_result; @@ -171,6 +175,11 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ + if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + ob_arg ? ob_arg : Py_None) < 0) { + return NULL; + } + if (ob_arg != NULL) { if (PyArg_Parse(ob_arg, "w*:ioctl", &pstr)) { char *arg; @@ -288,6 +297,10 @@ fcntl_flock_impl(PyObject *module, int fd, int code) int ret; int async_err = 0; + if (PySys_Audit("fcntl.flock", "ii", fd, code) < 0) { + return NULL; + } + #ifdef HAVE_FLOCK do { Py_BEGIN_ALLOW_THREADS @@ -372,6 +385,11 @@ fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, int ret; int async_err = 0; + if (PySys_Audit("fcntl.lockf", "iiOOi", fd, code, lenobj ? lenobj : Py_None, + startobj ? startobj : Py_None, whence) < 0) { + return NULL; + } + #ifndef LOCK_SH #define LOCK_SH 1 /* shared lock */ #define LOCK_EX 2 /* exclusive lock */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 88b47164bca09..2f791d1df953a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2827,6 +2827,10 @@ os_chdir_impl(PyObject *module, path_t *path) { int result; + if (PySys_Audit("os.chdir", "(O)", path->object) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef MS_WINDOWS /* on unix, success = 0, on windows, success = !0 */ @@ -2866,6 +2870,9 @@ static PyObject * os_fchdir_impl(PyObject *module, int fd) /*[clinic end generated code: output=42e064ec4dc00ab0 input=18e816479a2fa985]*/ { + if (PySys_Audit("os.chdir", "(i)", fd) < 0) { + return NULL; + } return posix_fildes_fd(fd, fchdir); } #endif /* HAVE_FCHDIR */ @@ -2923,6 +2930,11 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, return NULL; #endif + if (PySys_Audit("os.chmod", "Oii", path->object, mode, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS attr = GetFileAttributesW(path->wide); @@ -3019,6 +3031,10 @@ os_fchmod_impl(PyObject *module, int fd, int mode) int res; int async_err = 0; + if (PySys_Audit("os.chmod", "iii", fd, mode, -1) < 0) { + return NULL; + } + do { Py_BEGIN_ALLOW_THREADS res = fchmod(fd, mode); @@ -3050,6 +3066,9 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) /*[clinic end generated code: output=082344022b51a1d5 input=90c5663c7465d24f]*/ { int res; + if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { + return NULL; + } Py_BEGIN_ALLOW_THREADS res = lchmod(path->narrow, mode); Py_END_ALLOW_THREADS @@ -3092,6 +3111,10 @@ os_chflags_impl(PyObject *module, path_t *path, unsigned long flags, return NULL; #endif + if (PySys_Audit("os.chflags", "Ok", path->object, flags) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef HAVE_LCHFLAGS if (!follow_symlinks) @@ -3127,6 +3150,9 @@ os_lchflags_impl(PyObject *module, path_t *path, unsigned long flags) /*[clinic end generated code: output=30ae958695c07316 input=f9f82ea8b585ca9d]*/ { int res; + if (PySys_Audit("os.chflags", "Ok", path->object, flags) < 0) { + return NULL; + } Py_BEGIN_ALLOW_THREADS res = lchflags(path->narrow, flags); Py_END_ALLOW_THREADS @@ -3289,6 +3315,11 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, } #endif + if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef HAVE_FCHOWN if (path->fd != -1) @@ -3338,6 +3369,10 @@ os_fchown_impl(PyObject *module, int fd, uid_t uid, gid_t gid) int res; int async_err = 0; + if (PySys_Audit("os.chown", "iIIi", fd, uid, gid, -1) < 0) { + return NULL; + } + do { Py_BEGIN_ALLOW_THREADS res = fchown(fd, uid, gid); @@ -3370,6 +3405,9 @@ os_lchown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid) /*[clinic end generated code: output=25eaf6af412fdf2f input=b1c6014d563a7161]*/ { int res; + if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, -1) < 0) { + return NULL; + } Py_BEGIN_ALLOW_THREADS res = lchown(path->narrow, uid, gid); Py_END_ALLOW_THREADS @@ -3563,6 +3601,12 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, } #endif + if (PySys_Audit("os.link", "OOii", src->object, dst->object, + src_dir_fd == DEFAULT_DIR_FD ? -1 : src_dir_fd, + dst_dir_fd == DEFAULT_DIR_FD ? -1 : dst_dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS result = CreateHardLinkW(dst->wide, src->wide, NULL); @@ -4034,6 +4078,11 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) { int result; + if (PySys_Audit("os.mkdir", "Oii", path->object, mode, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS result = CreateDirectoryW(path->wide, NULL); @@ -4179,6 +4228,12 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is } #endif + if (PySys_Audit("os.rename", "OOii", src->object, dst->object, + src_dir_fd == DEFAULT_DIR_FD ? -1 : src_dir_fd, + dst_dir_fd == DEFAULT_DIR_FD ? -1 : dst_dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS result = MoveFileExW(src->wide, dst->wide, flags); @@ -4279,6 +4334,11 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) { int result; + if (PySys_Audit("os.rmdir", "Oi", path->object, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef MS_WINDOWS /* Windows, success=1, UNIX, success=0 */ @@ -4437,6 +4497,11 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) { int result; + if (PySys_Audit("os.remove", "Oi", path->object, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH #ifdef MS_WINDOWS @@ -4855,6 +4920,11 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, } #endif + if (PySys_Audit("os.utime", "OOOi", path->object, times, ns ? ns : Py_None, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS hFile = CreateFileW(path->wide, FILE_WRITE_ATTRIBUTES, 0, @@ -5156,8 +5226,7 @@ os_execv_impl(PyObject *module, path_t *path, PyObject *argv) return NULL; } - if (PySys_Audit("os.exec", "OOO", path->object ? path->object : Py_None, - argv, Py_None) < 0) { + if (PySys_Audit("os.exec", "OOO", path->object, argv, Py_None) < 0) { free_string_array(argvlist, argc); return NULL; } @@ -5233,8 +5302,7 @@ os_execve_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *env) if (envlist == NULL) goto fail_0; - if (PySys_Audit("os.exec", "OOO", path->object ? path->object : Py_None, - argv, env) < 0) { + if (PySys_Audit("os.exec", "OOO", path->object, argv, env) < 0) { goto fail_1; } @@ -5587,8 +5655,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a } attrp = &attr; - if (PySys_Audit("os.posix_spawn", "OOO", - path->object ? path->object : Py_None, argv, env) < 0) { + if (PySys_Audit("os.posix_spawn", "OOO", path->object, argv, env) < 0) { goto exit; } @@ -5832,8 +5899,7 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) mode = _P_OVERLAY; #endif - if (PySys_Audit("os.spawn", "iOOO", mode, - path->object ? path->object : Py_None, argv, + if (PySys_Audit("os.spawn", "iOOO", mode, path->object, argv, Py_None) < 0) { free_string_array(argvlist, argc); return NULL; @@ -5948,8 +6014,7 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, mode = _P_OVERLAY; #endif - if (PySys_Audit("os.spawn", "iOOO", mode, - path->object ? path->object : Py_None, argv, env) < 0) { + if (PySys_Audit("os.spawn", "iOOO", mode, path->object, argv, env) < 0) { goto fail_2; } @@ -6104,6 +6169,9 @@ os_fork_impl(PyObject *module) PyErr_SetString(PyExc_RuntimeError, "fork not supported for subinterpreters"); return NULL; } + if (PySys_Audit("os.fork", NULL) < 0) { + return NULL; + } PyOS_BeforeFork(); pid = fork(); if (pid == 0) { @@ -6709,6 +6777,9 @@ os_forkpty_impl(PyObject *module) PyErr_SetString(PyExc_RuntimeError, "fork not supported for subinterpreters"); return NULL; } + if (PySys_Audit("os.forkpty", NULL) < 0) { + return NULL; + } PyOS_BeforeFork(); pid = forkpty(&master_fd, NULL, NULL, NULL); if (pid == 0) { @@ -7230,14 +7301,15 @@ Kill a process with a signal. static PyObject * os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) /*[clinic end generated code: output=8e346a6701c88568 input=61a36b86ca275ab9]*/ -#ifndef MS_WINDOWS { + if (PySys_Audit("os.kill", "in", pid, signal) < 0) { + return NULL; + } +#ifndef MS_WINDOWS if (kill(pid, (int)signal) == -1) return posix_error(); Py_RETURN_NONE; -} #else /* !MS_WINDOWS */ -{ PyObject *result; DWORD sig = (DWORD)signal; DWORD err; @@ -7272,8 +7344,8 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) CloseHandle(handle); return result; -} #endif /* !MS_WINDOWS */ +} #endif /* HAVE_KILL */ @@ -7292,6 +7364,9 @@ static PyObject * os_killpg_impl(PyObject *module, pid_t pgid, int signal) /*[clinic end generated code: output=6dbcd2f1fdf5fdba input=38b5449eb8faec19]*/ { + if (PySys_Audit("os.killpg", "ii", pgid, signal) < 0) { + return NULL; + } /* XXX some man pages make the `pgid` parameter an int, others a pid_t. Since getpgrp() returns a pid_t, we assume killpg should take the same type. Moreover, pid_t is always at least as wide as @@ -8035,6 +8110,11 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, int result; #endif + if (PySys_Audit("os.symlink", "OOi", src->object, dst->object, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS if (windows_has_symlink_unprivileged_flag) { @@ -8638,6 +8718,10 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) { int res; + if (PySys_Audit("os.lockf", "iiL", fd, command, length) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS res = lockf(fd, command, length); Py_END_ALLOW_THREADS @@ -10070,6 +10154,9 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); return NULL; } + if (PySys_Audit("os.putenv", "OO", name, value) < 0) { + return NULL; + } bytes = PyBytes_FromFormat("%s=%s", name_string, value_string); if (bytes == NULL) { return NULL; @@ -10105,6 +10192,10 @@ os_unsetenv_impl(PyObject *module, PyObject *name) int err; #endif + if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { + return NULL; + } + #ifdef HAVE_BROKEN_UNSETENV unsetenv(PyBytes_AS_STRING(name)); #else @@ -11637,9 +11728,7 @@ os_startfile_impl(PyObject *module, path_t *filepath, "startfile not available on this platform"); } - if (PySys_Audit("os.startfile", "Ou", - filepath->object ? filepath->object : Py_None, - operation) < 0) { + if (PySys_Audit("os.startfile", "Ou", filepath->object, operation) < 0) { return NULL; } @@ -11817,6 +11906,10 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, if (fd_and_follow_symlinks_invalid("getxattr", path->fd, follow_symlinks)) return NULL; + if (PySys_Audit("os.getxattr", "OO", path->object, attribute->object) < 0) { + return NULL; + } + for (i = 0; ; i++) { void *ptr; ssize_t result; @@ -11888,6 +11981,11 @@ os_setxattr_impl(PyObject *module, path_t *path, path_t *attribute, if (fd_and_follow_symlinks_invalid("setxattr", path->fd, follow_symlinks)) return NULL; + if (PySys_Audit("os.setxattr", "OOy#i", path->object, attribute->object, + value->buf, value->len, flags) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS; if (path->fd > -1) result = fsetxattr(path->fd, attribute->narrow, @@ -11936,6 +12034,10 @@ os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, if (fd_and_follow_symlinks_invalid("removexattr", path->fd, follow_symlinks)) return NULL; + if (PySys_Audit("os.removexattr", "OO", path->object, attribute->object) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS; if (path->fd > -1) result = fremovexattr(path->fd, attribute->narrow); @@ -11981,6 +12083,11 @@ os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) if (fd_and_follow_symlinks_invalid("listxattr", path->fd, follow_symlinks)) goto exit; + if (PySys_Audit("os.listxattr", "(O)", + path->object ? path->object : Py_None) < 0) { + return NULL; + } + name = path->narrow ? path->narrow : "."; for (i = 0; ; i++) { @@ -13494,6 +13601,10 @@ os__add_dll_directory_impl(PyObject *module, path_t *path) DLL_DIRECTORY_COOKIE cookie = 0; DWORD err = 0; + if (PySys_Audit("os.add_dll_directory", "(O)", path->object) < 0) { + return NULL; + } + /* For Windows 7, we have to load this. As this will be a fairly infrequent operation, just do it each time. Kernel32 is always loaded. */ diff --git a/Modules/resource.c b/Modules/resource.c index 87c72e7409895..afde03c6c7e55 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -224,6 +224,11 @@ resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits) return NULL; } + if (PySys_Audit("resource.setrlimit", "iO", resource, + limits ? limits : Py_None) < 0) { + return NULL; + } + if (py2rlimit(limits, &rl) < 0) { return NULL; } @@ -269,6 +274,11 @@ resource_prlimit_impl(PyObject *module, pid_t pid, int resource, return NULL; } + if (PySys_Audit("resource.prlimit", "iiO", pid, resource, + limits ? limits : Py_None) < 0) { + return NULL; + } + if (group_right_1) { if (py2rlimit(limits, &new_limit) < 0) { return NULL; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 9aca70599689b..0c9a2671fe19b 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1233,6 +1233,10 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, { int err; + if (PySys_Audit("signal.pthread_kill", "ki", thread_id, signalnum) < 0) { + return NULL; + } + err = pthread_kill((pthread_t)thread_id, signalnum); if (err != 0) { errno = err; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index b2ea73baa1be4..66ea2703fc84a 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -144,6 +144,10 @@ syslog_openlog(PyObject * self, PyObject * args, PyObject *kwds) return NULL; } + if (PySys_Audit("syslog.openlog", "sll", ident, logopt, facility) < 0) { + return NULL; + } + openlog(ident, logopt, facility); S_log_open = 1; @@ -170,6 +174,10 @@ syslog_syslog(PyObject * self, PyObject * args) if (message == NULL) return NULL; + if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { + return NULL; + } + /* if log is not opened, open it now */ if (!S_log_open) { PyObject *openargs; @@ -194,6 +202,9 @@ syslog_syslog(PyObject * self, PyObject * args) static PyObject * syslog_closelog(PyObject *self, PyObject *unused) { + if (PySys_Audit("syslog.closelog", NULL) < 0) { + return NULL; + } if (S_log_open) { closelog(); Py_CLEAR(S_ident_o); @@ -209,6 +220,9 @@ syslog_setlogmask(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "l;mask for priority", &maskpri)) return NULL; + if (PySys_Audit("syslog.setlogmask", "(O)", args ? args : Py_None) < 0) { + return NULL; + } omaskpri = setlogmask(maskpri); return PyLong_FromLong(omaskpri); } diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index c4113e54c2b8f..5c06ec2621ea6 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -116,6 +116,10 @@ msvcrt_locking_impl(PyObject *module, int fd, int mode, long nbytes) { int err; + if (PySys_Audit("msvcrt.locking", "iil", fd, mode, nbytes) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH err = _locking(fd, mode, nbytes); @@ -175,6 +179,10 @@ msvcrt_open_osfhandle_impl(PyObject *module, void *handle, int flags) { int fd; + if (PySys_Audit("msvcrt.open_osfhandle", "Ki", handle, flags) < 0) { + return NULL; + } + _Py_BEGIN_SUPPRESS_IPH fd = _open_osfhandle((intptr_t)handle, flags); _Py_END_SUPPRESS_IPH @@ -201,6 +209,10 @@ msvcrt_get_osfhandle_impl(PyObject *module, int fd) { intptr_t handle = -1; + if (PySys_Audit("msvcrt.get_osfhandle", "(i)", fd) < 0) { + return NULL; + } + _Py_BEGIN_SUPPRESS_IPH handle = _get_osfhandle(fd); _Py_END_SUPPRESS_IPH

1 0

bpo-39606: allow closing async generators that are already closed (GH-18475)
by Nathaniel J. Smith 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/925dc7fb1d0db85dc137afa4cd14211bf0… commit: 925dc7fb1d0db85dc137afa4cd14211bf0d67414 branch: master author: Nathaniel J. Smith <njs(a)pobox.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T00:15:38-08:00 summary: bpo-39606: allow closing async generators that are already closed (GH-18475) The fix for [bpo-39386](https://bugs.python.org/issue39386) attempted to make it so you couldn't reuse a agen.aclose() coroutine object. It accidentally also prevented you from calling aclose() at all on an async generator that was already closed or exhausted. This commit fixes it so we're only blocking the actually illegal cases, while allowing the legal cases. The new tests failed before this patch. Also confirmed that this fixes the test failures we were seeing in Trio with Python dev builds: https://github.com/python-trio/trio/pull/1396 https://bugs.python.org/issue39606 files: A Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst M Lib/test/test_asyncgen.py M Objects/genobject.c diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 24b20bec2b2d1..fb6321d2264f3 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1128,7 +1128,7 @@ def exception_handler(loop, context): self.assertEqual([], messages) - def test_async_gen_await_anext_twice(self): + def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): yield 1 yield 2 @@ -1147,7 +1147,7 @@ def test_async_gen_await_anext_twice(self): self.loop.run_until_complete(run()) - def test_async_gen_await_aclose_twice(self): + def test_async_gen_await_same_aclose_coro_twice(self): async def async_iterate(): yield 1 yield 2 @@ -1164,6 +1164,32 @@ def test_async_gen_await_aclose_twice(self): self.loop.run_until_complete(run()) + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: + pass + await it.aclose() + + self.loop.run_until_complete(run()) if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst new file mode 100644 index 0000000000000..b7cbe4e91f59c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-02-11-23-59-07.bpo-39606.a72Sxc.rst @@ -0,0 +1,2 @@ +Fix regression caused by fix for bpo-39386, that prevented calling +``aclose`` on an async generator that had already been closed or exhausted. diff --git a/Objects/genobject.c b/Objects/genobject.c index 576d6856c7f30..0efd57de7a5a3 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1797,16 +1797,22 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) PyFrameObject *f = gen->gi_frame; PyObject *retval; - if (f == NULL || f->f_stacktop == NULL || - o->agt_state == AWAITABLE_STATE_CLOSED) { + if (o->agt_state == AWAITABLE_STATE_CLOSED) { PyErr_SetString( PyExc_RuntimeError, "cannot reuse already awaited aclose()/athrow()"); return NULL; } + if (f == NULL || f->f_stacktop == NULL) { + o->agt_state = AWAITABLE_STATE_CLOSED; + PyErr_SetNone(PyExc_StopIteration); + return NULL; + } + if (o->agt_state == AWAITABLE_STATE_INIT) { if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; if (o->agt_args == NULL) { PyErr_SetString( PyExc_RuntimeError, @@ -1878,7 +1884,6 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) /* aclose() mode */ if (retval) { if (_PyAsyncGenWrappedValue_CheckExact(retval)) { - o->agt_gen->ag_running_async = 0; Py_DECREF(retval); goto yield_close; } @@ -1893,16 +1898,17 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) yield_close: o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; check_error: o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - o->agt_state = AWAITABLE_STATE_CLOSED; if (o->agt_args == NULL) { /* when aclose() is called we don't want to propagate StopAsyncIteration or GeneratorExit; just raise @@ -1936,6 +1942,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *args) /* aclose() mode */ if (retval && _PyAsyncGenWrappedValue_CheckExact(retval)) { o->agt_gen->ag_running_async = 0; + o->agt_state = AWAITABLE_STATE_CLOSED; Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, ASYNC_GEN_IGNORED_EXIT_MSG); return NULL;

1 0

bpo-39184: Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, `shutil`, `signal`, `syslog` (GH-18407)
by Saiyang Gou 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/7514f4f6254f4a2d13ea8e5632a8e5f22b… commit: 7514f4f6254f4a2d13ea8e5632a8e5f22b637e0b branch: master author: Saiyang Gou <gousaiyang(a)163.com> committer: GitHub <noreply(a)github.com> date: 2020-02-13T07:47:42Z summary: bpo-39184: Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, `shutil`, `signal`, `syslog` (GH-18407) files: A Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst M Doc/library/fcntl.rst M Doc/library/msvcrt.rst M Doc/library/os.rst M Doc/library/resource.rst M Doc/library/shutil.rst M Doc/library/signal.rst M Doc/library/syslog.rst M Lib/shutil.py M Modules/fcntlmodule.c M Modules/posixmodule.c M Modules/resource.c M Modules/signalmodule.c M Modules/syslogmodule.c M PC/msvcrtmodule.c diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 5c172b836acca..07a15d27216e9 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -63,6 +63,8 @@ The module defines the following functions: If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. + .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl + .. function:: ioctl(fd, request, arg=0, mutate_flag=True) @@ -112,6 +114,8 @@ The module defines the following functions: >>> buf array('h', [13341]) + .. audit-event:: fcntl.ioctl fd,request,arg fcntl.ioctl + .. function:: flock(fd, operation) @@ -122,6 +126,8 @@ The module defines the following functions: If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. + .. audit-event:: fcntl.flock fd,operation fcntl.flock + .. function:: lockf(fd, cmd, len=0, start=0, whence=0) @@ -155,6 +161,8 @@ The module defines the following functions: The default for *len* is 0 which means to lock to the end of the file. The default for *whence* is also 0. + .. audit-event:: fcntl.lockf fd,cmd,len,start,whence fcntl.lockf + Examples (all on a SVR4 compliant system):: import struct, fcntl, os diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 14ad2cd4373af..42fffee6a0f44 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -42,6 +42,8 @@ File Operations regions in a file may be locked at the same time, but may not overlap. Adjacent regions are not merged; they must be unlocked individually. + .. audit-event:: msvcrt.locking fd,mode,nbytes msvcrt.locking + .. data:: LK_LOCK LK_RLCK @@ -77,12 +79,16 @@ File Operations and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. + .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle + .. function:: get_osfhandle(fd) Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if *fd* is not recognized. + .. audit-event:: msvcrt.get_osfhandle fd msvcrt.get_osfhandle + .. _msvcrt-console: diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b06a318c3d79d..af02a373f33dc 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -445,6 +445,8 @@ process and user. On some platforms, including FreeBSD and Mac OS X, setting ``environ`` may cause memory leaks. Refer to the system documentation for :c:func:`putenv`. + .. audit-event:: os.putenv key,value os.putenv + .. versionchanged:: 3.9 The function is now always available. @@ -640,6 +642,8 @@ process and user. don't update ``os.environ``, so it is actually preferable to delete items of ``os.environ``. + .. audit-event:: os.unsetenv key os.unsetenv + .. versionchanged:: 3.9 The function is now always available and is also available on Windows. @@ -766,6 +770,8 @@ as internal buffering of data. docs for :func:`chmod` for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(fd, mode)``. + .. audit-event:: os.chmod path,mode,dir_fd os.fchmod + .. availability:: Unix. @@ -776,6 +782,8 @@ as internal buffering of data. :func:`chown`. As of Python 3.3, this is equivalent to ``os.chown(fd, uid, gid)``. + .. audit-event:: os.chown path,uid,gid,dir_fd os.fchown + .. availability:: Unix. @@ -883,6 +891,8 @@ as internal buffering of data. :data:`F_ULOCK` or :data:`F_TEST`. *len* specifies the section of the file to lock. + .. audit-event:: os.lockf fd,cmd,len os.lockf + .. availability:: Unix. .. versionadded:: 3.3 @@ -1603,6 +1613,8 @@ features: This function can raise :exc:`OSError` and subclasses such as :exc:`FileNotFoundError`, :exc:`PermissionError`, and :exc:`NotADirectoryError`. + .. audit-event:: os.chdir path os.chdir + .. versionadded:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -1631,6 +1643,8 @@ features: This function can support :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.chflags path,flags os.chflags + .. availability:: Unix. .. versionadded:: 3.3 @@ -1676,6 +1690,8 @@ features: read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` constants or a corresponding integer value). All other bits are ignored. + .. audit-event:: os.chmod path,mode,dir_fd os.chmod + .. versionadded:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd* and *follow_symlinks* arguments. @@ -1696,6 +1712,8 @@ features: See :func:`shutil.chown` for a higher-level function that accepts names in addition to numeric ids. + .. audit-event:: os.chown path,uid,gid,dir_fd os.chown + .. availability:: Unix. .. versionadded:: 3.3 @@ -1722,6 +1740,8 @@ features: descriptor *fd*. The descriptor must refer to an opened directory, not an open file. As of Python 3.3, this is equivalent to ``os.chdir(fd)``. + .. audit-event:: os.chdir path os.fchdir + .. availability:: Unix. @@ -1746,6 +1766,8 @@ features: not follow symbolic links. As of Python 3.3, this is equivalent to ``os.chflags(path, flags, follow_symlinks=False)``. + .. audit-event:: os.chflags path,flags os.lchflags + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1759,6 +1781,8 @@ features: for possible values of *mode*. As of Python 3.3, this is equivalent to ``os.chmod(path, mode, follow_symlinks=False)``. + .. audit-event:: os.chmod path,mode,dir_fd os.lchmod + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1770,6 +1794,8 @@ features: function will not follow symbolic links. As of Python 3.3, this is equivalent to ``os.chown(path, uid, gid, follow_symlinks=False)``. + .. audit-event:: os.chown path,uid,gid,dir_fd os.lchown + .. availability:: Unix. .. versionchanged:: 3.6 @@ -1784,6 +1810,8 @@ features: supply :ref:`paths relative to directory descriptors <dir_fd>`, and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -1886,6 +1914,8 @@ features: It is also possible to create temporary directories; see the :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. + .. audit-event:: os.mkdir path,mode,dir_fd os.mkdir + .. versionadded:: 3.3 The *dir_fd* argument. @@ -1918,6 +1948,8 @@ features: This function handles UNC paths correctly. + .. audit-event:: os.mkdir path,mode,dir_fd os.makedirs + .. versionadded:: 3.2 The *exist_ok* parameter. @@ -2083,6 +2115,8 @@ features: This function is semantically identical to :func:`unlink`. + .. audit-event:: os.remove path,dir_fd os.remove + .. versionadded:: 3.3 The *dir_fd* argument. @@ -2103,6 +2137,8 @@ features: they are empty. Raises :exc:`OSError` if the leaf directory could not be successfully removed. + .. audit-event:: os.remove path,dir_fd os.removedirs + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -2128,6 +2164,8 @@ features: If you want cross-platform overwriting of the destination, use :func:`replace`. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.rename + .. versionadded:: 3.3 The *src_dir_fd* and *dst_dir_fd* arguments. @@ -2147,6 +2185,8 @@ features: This function can fail with the new directory structure made if you lack permissions needed to remove the leaf directory or file. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.renames + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *old* and *new*. @@ -2162,6 +2202,8 @@ features: This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to supply :ref:`paths relative to directory descriptors <dir_fd>`. + .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.replace + .. versionadded:: 3.3 .. versionchanged:: 3.6 @@ -2178,6 +2220,8 @@ features: This function can support :ref:`paths relative to directory descriptors <dir_fd>`. + .. audit-event:: os.rmdir path,dir_fd os.rmdir + .. versionadded:: 3.3 The *dir_fd* parameter. @@ -2821,6 +2865,8 @@ features: :exc:`OSError` is raised when the function is called by an unprivileged user. + .. audit-event:: os.symlink src,dst,dir_fd os.symlink + .. availability:: Unix, Windows. .. versionchanged:: 3.2 @@ -2873,6 +2919,8 @@ features: traditional Unix name. Please see the documentation for :func:`remove` for further information. + .. audit-event:: os.remove path,dir_fd os.unlink + .. versionadded:: 3.3 The *dir_fd* parameter. @@ -2910,6 +2958,8 @@ features: :ref:`paths relative to directory descriptors <dir_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.utime path,times,ns,dir_fd os.utime + .. versionadded:: 3.3 Added support for specifying *path* as an open file descriptor, and the *dir_fd*, *follow_symlinks*, and *ns* parameters. @@ -3135,6 +3185,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor <path_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.getxattr path,attribute os.getxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3149,6 +3201,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor <path_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.listxattr path os.listxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -3163,6 +3217,8 @@ These functions are all available on Linux only. This function can support :ref:`specifying a file descriptor <path_fd>` and :ref:`not following symlinks <follow_symlinks>`. + .. audit-event:: os.removexattr path,attribute os.removexattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3186,6 +3242,8 @@ These functions are all available on Linux only. A bug in Linux kernel versions less than 2.6.39 caused the flags argument to be ignored on some filesystems. + .. audit-event:: os.setxattr path,attribute,value,flags os.setxattr + .. versionchanged:: 3.6 Accepts a :term:`path-like object` for *path* and *attribute*. @@ -3248,6 +3306,8 @@ to be ignored. <https://msdn.microsoft.com/44228cf2-6306-466c-8f16-f513cd3ba8b5>`_ for more information about how DLLs are loaded. + .. audit-event:: os.add_dll_directory path os.add_dll_directory + .. availability:: Windows. .. versionadded:: 3.8 @@ -3480,6 +3540,8 @@ written in Python, such as a mail server's external command delivery program. Note that some platforms including FreeBSD <= 6.3 and Cygwin have known issues when using ``fork()`` from a thread. + .. audit-event:: os.fork "" os.fork + .. versionchanged:: 3.8 Calling ``fork()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -3499,6 +3561,8 @@ written in Python, such as a mail server's external command delivery program. master end of the pseudo-terminal. For a more portable approach, use the :mod:`pty` module. If an error occurs :exc:`OSError` is raised. + .. audit-event:: os.forkpty "" os.forkpty + .. versionchanged:: 3.8 Calling ``forkpty()`` in a subinterpreter is no longer supported (:exc:`RuntimeError` is raised). @@ -3525,6 +3589,8 @@ written in Python, such as a mail server's external command delivery program. See also :func:`signal.pthread_kill`. + .. audit-event:: os.kill pid,sig os.kill + .. versionadded:: 3.2 Windows support. @@ -3537,6 +3603,8 @@ written in Python, such as a mail server's external command delivery program. Send the signal *sig* to the process group *pgid*. + .. audit-event:: os.killpg pgid,sig os.killpg + .. availability:: Unix. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 3573da7ea2d71..e4eac43642d14 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -78,6 +78,9 @@ this module for those platforms. VxWorks only supports setting :data:`RLIMIT_NOFILE`. + .. audit-event:: resource.setrlimit resource,limits resource.setrlimit + + .. function:: prlimit(pid, resource[, limits]) Combines :func:`setrlimit` and :func:`getrlimit` in one function and @@ -94,6 +97,8 @@ this module for those platforms. :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for the process. + .. audit-event:: resource.prlimit pid,resource,limits resource.prlimit + .. availability:: Linux 2.6.36 or later with glibc 2.13 or later. .. versionadded:: 3.4 diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 59390d0e907eb..c7c63e6f80844 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -67,6 +67,8 @@ Directory and files operations a new symbolic link will be created instead of copying the file *src* points to. + .. audit-event:: shutil.copyfile src,dst shutil.copyfile + .. versionchanged:: 3.3 :exc:`IOError` used to be raised instead of :exc:`OSError`. Added *follow_symlinks* argument. @@ -101,6 +103,8 @@ Directory and files operations :func:`copymode` cannot modify symbolic links on the local platform, and it is asked to do so, it will do nothing and return. + .. audit-event:: shutil.copymode src,dst shutil.copymode + .. versionchanged:: 3.3 Added *follow_symlinks* argument. @@ -146,6 +150,8 @@ Directory and files operations Please see :data:`os.supports_follow_symlinks` for more information. + .. audit-event:: shutil.copystat src,dst shutil.copystat + .. versionchanged:: 3.3 Added *follow_symlinks* argument and support for Linux extended attributes. @@ -167,6 +173,10 @@ Directory and files operations To preserve all file metadata from the original, use :func:`~shutil.copy2` instead. + .. audit-event:: shutil.copyfile src,dst shutil.copy + + .. audit-event:: shutil.copymode src,dst shutil.copy + .. versionchanged:: 3.3 Added *follow_symlinks* argument. Now returns path to the newly created file. @@ -194,6 +204,10 @@ Directory and files operations Please see :func:`copystat` for more information about platform support for modifying symbolic link metadata. + .. audit-event:: shutil.copyfile src,dst shutil.copy2 + + .. audit-event:: shutil.copystat src,dst shutil.copy2 + .. versionchanged:: 3.3 Added *follow_symlinks* argument, try to copy extended file system attributes too (currently Linux only). @@ -342,6 +356,8 @@ Directory and files operations *copy_function* allows the move to succeed when it is not possible to also copy the metadata, at the expense of not copying any of the metadata. + .. audit-event:: shutil.move src,dst shutil.move + .. versionchanged:: 3.3 Added explicit symlink handling for foreign filesystems, thus adapting it to the behavior of GNU's :program:`mv`. @@ -381,6 +397,8 @@ Directory and files operations See also :func:`os.chown`, the underlying function. + .. audit-event:: shutil.chown path,user,group shutil.chown + .. availability:: Unix. .. versionadded:: 3.3 @@ -632,6 +650,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. registered for that extension. In case none is found, a :exc:`ValueError` is raised. + .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive + .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index a79fc501352d4..8b3ab412bd368 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -277,6 +277,8 @@ The :mod:`signal` module defines the following functions: If *signalnum* is 0, then no signal is sent, but error checking is still performed; this can be used to check if the target thread is still running. + .. audit-event:: signal.pthread_kill thread_id,signalnum signal.pthread_kill + .. availability:: Unix. See the man page :manpage:`pthread_kill(3)` for further information. diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 7151527ce57a5..d264a3340c98b 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -31,6 +31,8 @@ The module defines the following functions: If :func:`openlog` has not been called prior to the call to :func:`syslog`, ``openlog()`` will be called with no arguments. + .. audit-event:: syslog.syslog priority,message syslog.syslog + .. function:: openlog([ident[, logoption[, facility]]]) @@ -45,6 +47,8 @@ The module defines the following functions: keyword argument (default is :const:`LOG_USER`) sets the default facility for messages which do not have a facility explicitly encoded. + .. audit-event:: syslog.openlog ident,logoption,facility syslog.openlog + .. versionchanged:: 3.2 In previous versions, keyword arguments were not allowed, and *ident* was required. The default for *ident* was dependent on the system libraries, @@ -60,6 +64,8 @@ The module defines the following functions: :func:`openlog` hasn't already been called), and *ident* and other :func:`openlog` parameters are reset to defaults. + .. audit-event:: syslog.closelog "" syslog.closelog + .. function:: setlogmask(maskpri) @@ -70,6 +76,8 @@ The module defines the following functions: ``LOG_UPTO(pri)`` calculates the mask for all priorities up to and including *pri*. + .. audit-event:: syslog.setlogmask maskpri syslog.setlogmask + The module defines the following constants: Priority levels (high to low): diff --git a/Lib/shutil.py b/Lib/shutil.py index 9a83a3242ed91..a4ce2c0290bc9 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -235,6 +235,8 @@ def copyfile(src, dst, *, follow_symlinks=True): symlink will be created instead of copying the file it points to. """ + sys.audit("shutil.copyfile", src, dst) + if _samefile(src, dst): raise SameFileError("{!r} and {!r} are the same file".format(src, dst)) @@ -289,6 +291,8 @@ def copymode(src, dst, *, follow_symlinks=True): (e.g. Linux) this method does nothing. """ + sys.audit("shutil.copymode", src, dst) + if not follow_symlinks and _islink(src) and os.path.islink(dst): if hasattr(os, 'lchmod'): stat_func, chmod_func = os.lstat, os.lchmod @@ -340,6 +344,8 @@ def copystat(src, dst, *, follow_symlinks=True): If the optional flag `follow_symlinks` is not set, symlinks aren't followed if and only if both `src` and `dst` are symlinks. """ + sys.audit("shutil.copystat", src, dst) + def _nop(*args, ns=None, follow_symlinks=None): pass @@ -778,6 +784,7 @@ def move(src, dst, copy_function=copy2): the issues this implementation glosses over. """ + sys.audit("shutil.move", src, dst) real_dst = dst if os.path.isdir(dst): if _samefile(src, dst): @@ -1208,6 +1215,8 @@ def unpack_archive(filename, extract_dir=None, format=None): In case none is found, a ValueError is raised. """ + sys.audit("shutil.unpack_archive", filename, extract_dir, format) + if extract_dir is None: extract_dir = os.getcwd() @@ -1275,6 +1284,7 @@ def chown(path, user=None, group=None): user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. """ + sys.audit('shutil.chown', path, user, group) if user is None and group is None: raise ValueError("user and/or group must be set") diff --git a/Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst b/Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst new file mode 100644 index 0000000000000..cf25c24d58788 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-02-07-23-54-18.bpo-39184.v-ue-v.rst @@ -0,0 +1 @@ +Add audit events to functions in `fcntl`, `msvcrt`, `os`, `resource`, `shutil`, `signal` and `syslog`. \ No newline at end of file diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 11906aa582929..43f9b22f67207 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -66,6 +66,10 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) char buf[1024]; int async_err = 0; + if (PySys_Audit("fcntl.fcntl", "iiO", fd, code, arg ? arg : Py_None) < 0) { + return NULL; + } + if (arg != NULL) { int parse_result; @@ -171,6 +175,11 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ + if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + ob_arg ? ob_arg : Py_None) < 0) { + return NULL; + } + if (ob_arg != NULL) { if (PyArg_Parse(ob_arg, "w*:ioctl", &pstr)) { char *arg; @@ -288,6 +297,10 @@ fcntl_flock_impl(PyObject *module, int fd, int code) int ret; int async_err = 0; + if (PySys_Audit("fcntl.flock", "ii", fd, code) < 0) { + return NULL; + } + #ifdef HAVE_FLOCK do { Py_BEGIN_ALLOW_THREADS @@ -372,6 +385,11 @@ fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, int ret; int async_err = 0; + if (PySys_Audit("fcntl.lockf", "iiOOi", fd, code, lenobj ? lenobj : Py_None, + startobj ? startobj : Py_None, whence) < 0) { + return NULL; + } + #ifndef LOCK_SH #define LOCK_SH 1 /* shared lock */ #define LOCK_EX 2 /* exclusive lock */ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ec3da4fb2fc6e..4d6d255b3469b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2911,6 +2911,10 @@ os_chdir_impl(PyObject *module, path_t *path) { int result; + if (PySys_Audit("os.chdir", "(O)", path->object) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef MS_WINDOWS /* on unix, success = 0, on windows, success = !0 */ @@ -2950,6 +2954,9 @@ static PyObject * os_fchdir_impl(PyObject *module, int fd) /*[clinic end generated code: output=42e064ec4dc00ab0 input=18e816479a2fa985]*/ { + if (PySys_Audit("os.chdir", "(i)", fd) < 0) { + return NULL; + } return posix_fildes_fd(fd, fchdir); } #endif /* HAVE_FCHDIR */ @@ -3007,6 +3014,11 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, return NULL; #endif + if (PySys_Audit("os.chmod", "Oii", path->object, mode, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS attr = GetFileAttributesW(path->wide); @@ -3103,6 +3115,10 @@ os_fchmod_impl(PyObject *module, int fd, int mode) int res; int async_err = 0; + if (PySys_Audit("os.chmod", "iii", fd, mode, -1) < 0) { + return NULL; + } + do { Py_BEGIN_ALLOW_THREADS res = fchmod(fd, mode); @@ -3134,6 +3150,9 @@ os_lchmod_impl(PyObject *module, path_t *path, int mode) /*[clinic end generated code: output=082344022b51a1d5 input=90c5663c7465d24f]*/ { int res; + if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { + return NULL; + } Py_BEGIN_ALLOW_THREADS res = lchmod(path->narrow, mode); Py_END_ALLOW_THREADS @@ -3176,6 +3195,10 @@ os_chflags_impl(PyObject *module, path_t *path, unsigned long flags, return NULL; #endif + if (PySys_Audit("os.chflags", "Ok", path->object, flags) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef HAVE_LCHFLAGS if (!follow_symlinks) @@ -3211,6 +3234,9 @@ os_lchflags_impl(PyObject *module, path_t *path, unsigned long flags) /*[clinic end generated code: output=30ae958695c07316 input=f9f82ea8b585ca9d]*/ { int res; + if (PySys_Audit("os.chflags", "Ok", path->object, flags) < 0) { + return NULL; + } Py_BEGIN_ALLOW_THREADS res = lchflags(path->narrow, flags); Py_END_ALLOW_THREADS @@ -3373,6 +3399,11 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, } #endif + if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef HAVE_FCHOWN if (path->fd != -1) @@ -3422,6 +3453,10 @@ os_fchown_impl(PyObject *module, int fd, uid_t uid, gid_t gid) int res; int async_err = 0; + if (PySys_Audit("os.chown", "iIIi", fd, uid, gid, -1) < 0) { + return NULL; + } + do { Py_BEGIN_ALLOW_THREADS res = fchown(fd, uid, gid); @@ -3454,6 +3489,9 @@ os_lchown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid) /*[clinic end generated code: output=25eaf6af412fdf2f input=b1c6014d563a7161]*/ { int res; + if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, -1) < 0) { + return NULL; + } Py_BEGIN_ALLOW_THREADS res = lchown(path->narrow, uid, gid); Py_END_ALLOW_THREADS @@ -3647,6 +3685,12 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, } #endif + if (PySys_Audit("os.link", "OOii", src->object, dst->object, + src_dir_fd == DEFAULT_DIR_FD ? -1 : src_dir_fd, + dst_dir_fd == DEFAULT_DIR_FD ? -1 : dst_dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS result = CreateHardLinkW(dst->wide, src->wide, NULL); @@ -4114,6 +4158,11 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) { int result; + if (PySys_Audit("os.mkdir", "Oii", path->object, mode, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS result = CreateDirectoryW(path->wide, NULL); @@ -4259,6 +4308,12 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is } #endif + if (PySys_Audit("os.rename", "OOii", src->object, dst->object, + src_dir_fd == DEFAULT_DIR_FD ? -1 : src_dir_fd, + dst_dir_fd == DEFAULT_DIR_FD ? -1 : dst_dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS result = MoveFileExW(src->wide, dst->wide, flags); @@ -4359,6 +4414,11 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) { int result; + if (PySys_Audit("os.rmdir", "Oi", path->object, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS #ifdef MS_WINDOWS /* Windows, success=1, UNIX, success=0 */ @@ -4517,6 +4577,11 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) { int result; + if (PySys_Audit("os.remove", "Oi", path->object, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH #ifdef MS_WINDOWS @@ -4933,6 +4998,11 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, } #endif + if (PySys_Audit("os.utime", "OOOi", path->object, times, ns ? ns : Py_None, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS hFile = CreateFileW(path->wide, FILE_WRITE_ATTRIBUTES, 0, @@ -5234,8 +5304,7 @@ os_execv_impl(PyObject *module, path_t *path, PyObject *argv) return NULL; } - if (PySys_Audit("os.exec", "OOO", path->object ? path->object : Py_None, - argv, Py_None) < 0) { + if (PySys_Audit("os.exec", "OOO", path->object, argv, Py_None) < 0) { free_string_array(argvlist, argc); return NULL; } @@ -5311,8 +5380,7 @@ os_execve_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *env) if (envlist == NULL) goto fail_0; - if (PySys_Audit("os.exec", "OOO", path->object ? path->object : Py_None, - argv, env) < 0) { + if (PySys_Audit("os.exec", "OOO", path->object, argv, env) < 0) { goto fail_1; } @@ -5665,8 +5733,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a } attrp = &attr; - if (PySys_Audit("os.posix_spawn", "OOO", - path->object ? path->object : Py_None, argv, env) < 0) { + if (PySys_Audit("os.posix_spawn", "OOO", path->object, argv, env) < 0) { goto exit; } @@ -5910,8 +5977,7 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) mode = _P_OVERLAY; #endif - if (PySys_Audit("os.spawn", "iOOO", mode, - path->object ? path->object : Py_None, argv, + if (PySys_Audit("os.spawn", "iOOO", mode, path->object, argv, Py_None) < 0) { free_string_array(argvlist, argc); return NULL; @@ -6026,8 +6092,7 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, mode = _P_OVERLAY; #endif - if (PySys_Audit("os.spawn", "iOOO", mode, - path->object ? path->object : Py_None, argv, env) < 0) { + if (PySys_Audit("os.spawn", "iOOO", mode, path->object, argv, env) < 0) { goto fail_2; } @@ -6182,6 +6247,9 @@ os_fork_impl(PyObject *module) PyErr_SetString(PyExc_RuntimeError, "fork not supported for subinterpreters"); return NULL; } + if (PySys_Audit("os.fork", NULL) < 0) { + return NULL; + } PyOS_BeforeFork(); pid = fork(); if (pid == 0) { @@ -6788,6 +6856,9 @@ os_forkpty_impl(PyObject *module) PyErr_SetString(PyExc_RuntimeError, "fork not supported for subinterpreters"); return NULL; } + if (PySys_Audit("os.forkpty", NULL) < 0) { + return NULL; + } PyOS_BeforeFork(); pid = forkpty(&master_fd, NULL, NULL, NULL); if (pid == 0) { @@ -7309,14 +7380,15 @@ Kill a process with a signal. static PyObject * os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) /*[clinic end generated code: output=8e346a6701c88568 input=61a36b86ca275ab9]*/ -#ifndef MS_WINDOWS { + if (PySys_Audit("os.kill", "in", pid, signal) < 0) { + return NULL; + } +#ifndef MS_WINDOWS if (kill(pid, (int)signal) == -1) return posix_error(); Py_RETURN_NONE; -} #else /* !MS_WINDOWS */ -{ PyObject *result; DWORD sig = (DWORD)signal; DWORD err; @@ -7351,8 +7423,8 @@ os_kill_impl(PyObject *module, pid_t pid, Py_ssize_t signal) CloseHandle(handle); return result; -} #endif /* !MS_WINDOWS */ +} #endif /* HAVE_KILL */ @@ -7371,6 +7443,9 @@ static PyObject * os_killpg_impl(PyObject *module, pid_t pgid, int signal) /*[clinic end generated code: output=6dbcd2f1fdf5fdba input=38b5449eb8faec19]*/ { + if (PySys_Audit("os.killpg", "ii", pgid, signal) < 0) { + return NULL; + } /* XXX some man pages make the `pgid` parameter an int, others a pid_t. Since getpgrp() returns a pid_t, we assume killpg should take the same type. Moreover, pid_t is always at least as wide as @@ -8143,6 +8218,11 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, int result; #endif + if (PySys_Audit("os.symlink", "OOi", src->object, dst->object, + dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { + return NULL; + } + #ifdef MS_WINDOWS if (windows_has_symlink_unprivileged_flag) { @@ -8745,6 +8825,10 @@ os_lockf_impl(PyObject *module, int fd, int command, Py_off_t length) { int res; + if (PySys_Audit("os.lockf", "iiL", fd, command, length) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS res = lockf(fd, command, length); Py_END_ALLOW_THREADS @@ -10147,6 +10231,9 @@ static PyObject * os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic end generated code: output=d29a567d6b2327d2 input=ba586581c2e6105f]*/ { + if (PySys_Audit("os.putenv", "OO", name, value) < 0) { + return NULL; + } return win32_putenv(name, value); } #else @@ -10172,6 +10259,10 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) return NULL; } + if (PySys_Audit("os.putenv", "OO", name, value) < 0) { + return NULL; + } + if (setenv(name_string, value_string, 1)) { return posix_error(); } @@ -10193,6 +10284,9 @@ static PyObject * os_unsetenv_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=54c4137ab1834f02 input=4d6a1747cc526d2f]*/ { + if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { + return NULL; + } return win32_putenv(name, NULL); } #else @@ -10208,6 +10302,9 @@ static PyObject * os_unsetenv_impl(PyObject *module, PyObject *name) /*[clinic end generated code: output=54c4137ab1834f02 input=2bb5288a599c7107]*/ { + if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { + return NULL; + } #ifdef HAVE_BROKEN_UNSETENV unsetenv(PyBytes_AS_STRING(name)); #else @@ -11730,9 +11827,7 @@ os_startfile_impl(PyObject *module, path_t *filepath, "startfile not available on this platform"); } - if (PySys_Audit("os.startfile", "Ou", - filepath->object ? filepath->object : Py_None, - operation) < 0) { + if (PySys_Audit("os.startfile", "Ou", filepath->object, operation) < 0) { return NULL; } @@ -11910,6 +12005,10 @@ os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, if (fd_and_follow_symlinks_invalid("getxattr", path->fd, follow_symlinks)) return NULL; + if (PySys_Audit("os.getxattr", "OO", path->object, attribute->object) < 0) { + return NULL; + } + for (i = 0; ; i++) { void *ptr; ssize_t result; @@ -11981,6 +12080,11 @@ os_setxattr_impl(PyObject *module, path_t *path, path_t *attribute, if (fd_and_follow_symlinks_invalid("setxattr", path->fd, follow_symlinks)) return NULL; + if (PySys_Audit("os.setxattr", "OOy#i", path->object, attribute->object, + value->buf, value->len, flags) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS; if (path->fd > -1) result = fsetxattr(path->fd, attribute->narrow, @@ -12029,6 +12133,10 @@ os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, if (fd_and_follow_symlinks_invalid("removexattr", path->fd, follow_symlinks)) return NULL; + if (PySys_Audit("os.removexattr", "OO", path->object, attribute->object) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS; if (path->fd > -1) result = fremovexattr(path->fd, attribute->narrow); @@ -12074,6 +12182,11 @@ os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) if (fd_and_follow_symlinks_invalid("listxattr", path->fd, follow_symlinks)) goto exit; + if (PySys_Audit("os.listxattr", "(O)", + path->object ? path->object : Py_None) < 0) { + return NULL; + } + name = path->narrow ? path->narrow : "."; for (i = 0; ; i++) { @@ -13550,6 +13663,10 @@ os__add_dll_directory_impl(PyObject *module, path_t *path) DLL_DIRECTORY_COOKIE cookie = 0; DWORD err = 0; + if (PySys_Audit("os.add_dll_directory", "(O)", path->object) < 0) { + return NULL; + } + /* For Windows 7, we have to load this. As this will be a fairly infrequent operation, just do it each time. Kernel32 is always loaded. */ diff --git a/Modules/resource.c b/Modules/resource.c index 87c72e7409895..afde03c6c7e55 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -224,6 +224,11 @@ resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits) return NULL; } + if (PySys_Audit("resource.setrlimit", "iO", resource, + limits ? limits : Py_None) < 0) { + return NULL; + } + if (py2rlimit(limits, &rl) < 0) { return NULL; } @@ -269,6 +274,11 @@ resource_prlimit_impl(PyObject *module, pid_t pid, int resource, return NULL; } + if (PySys_Audit("resource.prlimit", "iiO", pid, resource, + limits ? limits : Py_None) < 0) { + return NULL; + } + if (group_right_1) { if (py2rlimit(limits, &new_limit) < 0) { return NULL; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 693b90b6c631e..a197673746240 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1236,6 +1236,10 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, { int err; + if (PySys_Audit("signal.pthread_kill", "ki", thread_id, signalnum) < 0) { + return NULL; + } + err = pthread_kill((pthread_t)thread_id, signalnum); if (err != 0) { errno = err; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 539224f2c5b67..24517925c32eb 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -144,6 +144,10 @@ syslog_openlog(PyObject * self, PyObject * args, PyObject *kwds) return NULL; } + if (PySys_Audit("syslog.openlog", "sll", ident, logopt, facility) < 0) { + return NULL; + } + openlog(ident, logopt, facility); S_log_open = 1; @@ -170,6 +174,10 @@ syslog_syslog(PyObject * self, PyObject * args) if (message == NULL) return NULL; + if (PySys_Audit("syslog.syslog", "is", priority, message) < 0) { + return NULL; + } + /* if log is not opened, open it now */ if (!S_log_open) { PyObject *openargs; @@ -194,6 +202,9 @@ syslog_syslog(PyObject * self, PyObject * args) static PyObject * syslog_closelog(PyObject *self, PyObject *unused) { + if (PySys_Audit("syslog.closelog", NULL) < 0) { + return NULL; + } if (S_log_open) { closelog(); Py_CLEAR(S_ident_o); @@ -209,6 +220,9 @@ syslog_setlogmask(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "l;mask for priority", &maskpri)) return NULL; + if (PySys_Audit("syslog.setlogmask", "(O)", args ? args : Py_None) < 0) { + return NULL; + } omaskpri = setlogmask(maskpri); return PyLong_FromLong(omaskpri); } diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index c4113e54c2b8f..5c06ec2621ea6 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -116,6 +116,10 @@ msvcrt_locking_impl(PyObject *module, int fd, int mode, long nbytes) { int err; + if (PySys_Audit("msvcrt.locking", "iil", fd, mode, nbytes) < 0) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH err = _locking(fd, mode, nbytes); @@ -175,6 +179,10 @@ msvcrt_open_osfhandle_impl(PyObject *module, void *handle, int flags) { int fd; + if (PySys_Audit("msvcrt.open_osfhandle", "Ki", handle, flags) < 0) { + return NULL; + } + _Py_BEGIN_SUPPRESS_IPH fd = _open_osfhandle((intptr_t)handle, flags); _Py_END_SUPPRESS_IPH @@ -201,6 +209,10 @@ msvcrt_get_osfhandle_impl(PyObject *module, int fd) { intptr_t handle = -1; + if (PySys_Audit("msvcrt.get_osfhandle", "(i)", fd) < 0) { + return NULL; + } + _Py_BEGIN_SUPPRESS_IPH handle = _get_osfhandle(fd); _Py_END_SUPPRESS_IPH

1 0

closes bpo-39621: Make buf arg to md5_compress be const. (GH-18497)
by Miss Islington (bot) 13 Feb '20

13 Feb '20

https://github.com/python/cpython/commit/669981b3b14dd16dec42089d6ac8d6449f… commit: 669981b3b14dd16dec42089d6ac8d6449fde8abd branch: 3.8 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: GitHub <noreply(a)github.com> date: 2020-02-12T21:12:53-08:00 summary: closes bpo-39621: Make buf arg to md5_compress be const. (GH-18497) (cherry picked from commit 597ebed748d0b0c061f8c108bd98270d103286c1) Co-authored-by: Andy Lester <andy(a)petdance.com> files: M Modules/md5module.c diff --git a/Modules/md5module.c b/Modules/md5module.c index b9a351a8c1cdd..c2ebaaf61f91c 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -119,7 +119,7 @@ typedef struct { a = (a + I(b,c,d) + M + t); a = ROLc(a, s) + b; -static void md5_compress(struct md5_state *md5, unsigned char *buf) +static void md5_compress(struct md5_state *md5, const unsigned char *buf) { MD5_INT32 i, W[16], a, b, c, d; @@ -242,7 +242,7 @@ md5_process(struct md5_state *md5, const unsigned char *in, Py_ssize_t inlen) while (inlen > 0) { if (md5->curlen == 0 && inlen >= MD5_BLOCKSIZE) { - md5_compress(md5, (unsigned char *)in); + md5_compress(md5, in); md5->length += MD5_BLOCKSIZE * 8; in += MD5_BLOCKSIZE; inlen -= MD5_BLOCKSIZE;

1 0