[Python-checkins] Improve object stats (#92845)
markshannon
webhook-mailer at python.org
Mon May 16 09:35:15 EDT 2022
https://github.com/python/cpython/commit/fa2b8b75eb2b8a0193d587e02b488a73579118fc
commit: fa2b8b75eb2b8a0193d587e02b488a73579118fc
branch: main
author: Mark Shannon <mark at hotpy.org>
committer: markshannon <mark at hotpy.org>
date: 2022-05-16T14:35:11+01:00
summary:
Improve object stats (#92845)
* Add incref/decref stats
* Show ratios for allocation in summary
files:
A Include/pystats.h
M Include/internal/pycore_code.h
M Include/internal/pycore_object.h
M Include/object.h
M Makefile.pre.in
M PCbuild/pythoncore.vcxproj
M PCbuild/pythoncore.vcxproj.filters
M Python/ceval.c
M Python/specialize.c
M Tools/scripts/summarize_stats.py
diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h
index e11d1f05129c6..96cc9dcae11a7 100644
--- a/Include/internal/pycore_code.h
+++ b/Include/internal/pycore_code.h
@@ -265,53 +265,6 @@ extern int _PyStaticCode_InternStrings(PyCodeObject *co);
#ifdef Py_STATS
-#define SPECIALIZATION_FAILURE_KINDS 30
-
-typedef struct _specialization_stats {
- uint64_t success;
- uint64_t failure;
- uint64_t hit;
- uint64_t deferred;
- uint64_t miss;
- uint64_t deopt;
- uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
-} SpecializationStats;
-
-typedef struct _opcode_stats {
- SpecializationStats specialization;
- uint64_t execution_count;
- uint64_t pair_count[256];
-} OpcodeStats;
-
-typedef struct _call_stats {
- uint64_t inlined_py_calls;
- uint64_t pyeval_calls;
- uint64_t frames_pushed;
- uint64_t frame_objects_created;
-} CallStats;
-
-typedef struct _object_stats {
- uint64_t allocations;
- uint64_t allocations512;
- uint64_t allocations4k;
- uint64_t allocations_big;
- uint64_t frees;
- uint64_t to_freelist;
- uint64_t from_freelist;
- uint64_t new_values;
- uint64_t dict_materialized_on_request;
- uint64_t dict_materialized_new_key;
- uint64_t dict_materialized_too_big;
- uint64_t dict_materialized_str_subclass;
-} ObjectStats;
-
-typedef struct _stats {
- OpcodeStats opcode_stats[256];
- CallStats call_stats;
- ObjectStats object_stats;
-} PyStats;
-
-extern PyStats _py_stats;
#define STAT_INC(opname, name) _py_stats.opcode_stats[opname].specialization.name++
#define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
@@ -321,8 +274,6 @@ extern PyStats _py_stats;
#define OBJECT_STAT_INC_COND(name, cond) \
do { if (cond) _py_stats.object_stats.name++; } while (0)
-extern void _Py_PrintSpecializationStats(int to_file);
-
// Used by the _opcode extension which is built as a shared library
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h
index f022f8246989d..cc50418e2ef40 100644
--- a/Include/internal/pycore_object.h
+++ b/Include/internal/pycore_object.h
@@ -34,6 +34,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
static inline void
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
{
+ _Py_DECREF_STAT_INC();
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif
@@ -51,6 +52,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
static inline void
_Py_DECREF_NO_DEALLOC(PyObject *op)
{
+ _Py_DECREF_STAT_INC();
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif
diff --git a/Include/object.h b/Include/object.h
index f2af428e2bb97..f01b9fa86d014 100644
--- a/Include/object.h
+++ b/Include/object.h
@@ -51,6 +51,8 @@ A standard interface exists for objects that contain an array of items
whose size is determined when the object is allocated.
*/
+#include "pystats.h"
+
/* Py_DEBUG implies Py_REF_DEBUG. */
#if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
# define Py_REF_DEBUG
@@ -490,6 +492,7 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *);
static inline void Py_INCREF(PyObject *op)
{
+ _Py_INCREF_STAT_INC();
#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
// Stable ABI for Python 3.10 built in debug mode.
_Py_IncRef(op);
@@ -506,7 +509,6 @@ static inline void Py_INCREF(PyObject *op)
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
#endif
-
#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
// Stable ABI for limited C API version 3.10 of Python debug build
static inline void Py_DECREF(PyObject *op) {
@@ -517,6 +519,7 @@ static inline void Py_DECREF(PyObject *op) {
#elif defined(Py_REF_DEBUG)
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
{
+ _Py_DECREF_STAT_INC();
_Py_RefTotal--;
if (--op->ob_refcnt != 0) {
if (op->ob_refcnt < 0) {
@@ -532,6 +535,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
#else
static inline void Py_DECREF(PyObject *op)
{
+ _Py_DECREF_STAT_INC();
// Non-limited C API and limited C API for Python 3.9 and older access
// directly PyObject.ob_refcnt.
if (--op->ob_refcnt == 0) {
diff --git a/Include/pystats.h b/Include/pystats.h
new file mode 100644
index 0000000000000..bc05dd864c63a
--- /dev/null
+++ b/Include/pystats.h
@@ -0,0 +1,77 @@
+
+
+#ifndef Py_PYSTATS_H
+#define Py_PYSTATS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef Py_STATS
+
+#define SPECIALIZATION_FAILURE_KINDS 32
+
+typedef struct _specialization_stats {
+ uint64_t success;
+ uint64_t failure;
+ uint64_t hit;
+ uint64_t deferred;
+ uint64_t miss;
+ uint64_t deopt;
+ uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
+} SpecializationStats;
+
+typedef struct _opcode_stats {
+ SpecializationStats specialization;
+ uint64_t execution_count;
+ uint64_t pair_count[256];
+} OpcodeStats;
+
+typedef struct _call_stats {
+ uint64_t inlined_py_calls;
+ uint64_t pyeval_calls;
+ uint64_t frames_pushed;
+ uint64_t frame_objects_created;
+} CallStats;
+
+typedef struct _object_stats {
+ uint64_t increfs;
+ uint64_t decrefs;
+ uint64_t allocations;
+ uint64_t allocations512;
+ uint64_t allocations4k;
+ uint64_t allocations_big;
+ uint64_t frees;
+ uint64_t to_freelist;
+ uint64_t from_freelist;
+ uint64_t new_values;
+ uint64_t dict_materialized_on_request;
+ uint64_t dict_materialized_new_key;
+ uint64_t dict_materialized_too_big;
+ uint64_t dict_materialized_str_subclass;
+} ObjectStats;
+
+typedef struct _stats {
+ OpcodeStats opcode_stats[256];
+ CallStats call_stats;
+ ObjectStats object_stats;
+} PyStats;
+
+PyAPI_DATA(PyStats) _py_stats;
+
+extern void _Py_PrintSpecializationStats(int to_file);
+
+
+#define _Py_INCREF_STAT_INC() _py_stats.object_stats.increfs++
+#define _Py_DECREF_STAT_INC() _py_stats.object_stats.decrefs++
+
+#else
+
+#define _Py_INCREF_STAT_INC() ((void)0)
+#define _Py_DECREF_STAT_INC() ((void)0)
+
+#endif // !Py_STATS
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PYSTATs_H */
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 515c18cc21666..a178dd0a65c73 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -1509,6 +1509,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/pymem.h \
$(srcdir)/Include/pyport.h \
$(srcdir)/Include/pystate.h \
+ $(srcdir)/Include/pystats.h \
$(srcdir)/Include/pystrcmp.h \
$(srcdir)/Include/pystrtod.h \
$(srcdir)/Include/pythonrun.h \
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index a35884b3c3588..be76f1bc55859 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -280,6 +280,7 @@
<ClInclude Include="..\Include\pymem.h" />
<ClInclude Include="..\Include\pyport.h" />
<ClInclude Include="..\Include\pystate.h" />
+ <ClInclude Include="..\Include\pystats.h" />
<ClInclude Include="..\Include\pystrcmp.h" />
<ClInclude Include="..\Include\pystrtod.h" />
<ClInclude Include="..\Include\pythonrun.h" />
diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters
index ff42cc92c4bd2..5573e0020491a 100644
--- a/PCbuild/pythoncore.vcxproj.filters
+++ b/PCbuild/pythoncore.vcxproj.filters
@@ -174,6 +174,9 @@
<ClInclude Include="..\Include\pystate.h">
<Filter>Include</Filter>
</ClInclude>
+ <ClInclude Include="..\Include\pystats.h">
+ <Filter>Include</Filter>
+ </ClInclude>
<ClInclude Include="..\Include\pystrcmp.h">
<Filter>Include</Filter>
</ClInclude>
diff --git a/Python/ceval.c b/Python/ceval.c
index c73218fcf307e..c81d0efff9b9b 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -55,6 +55,7 @@
#undef Py_DECREF
#define Py_DECREF(arg) \
do { \
+ _Py_DECREF_STAT_INC(); \
PyObject *op = _PyObject_CAST(arg); \
if (--op->ob_refcnt == 0) { \
destructor dealloc = Py_TYPE(op)->tp_dealloc; \
@@ -78,6 +79,7 @@
#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
do { \
+ _Py_DECREF_STAT_INC(); \
PyObject *op = _PyObject_CAST(arg); \
if (--op->ob_refcnt == 0) { \
destructor d = (destructor)(dealloc); \
diff --git a/Python/specialize.c b/Python/specialize.c
index fa42993606fd6..6a91389f85610 100644
--- a/Python/specialize.c
+++ b/Python/specialize.c
@@ -191,6 +191,8 @@ print_object_stats(FILE *out, ObjectStats *stats)
fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
+ fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
+ fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index bc528ca316f93..91b190114008e 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -8,6 +8,7 @@
from datetime import date
import itertools
import argparse
+import sys
if os.name == "nt":
DEFAULT_DIR = "c:\\temp\\py_stats\\"
@@ -88,7 +89,11 @@ def gather_stats():
for filename in os.listdir(DEFAULT_DIR):
with open(os.path.join(DEFAULT_DIR, filename)) as fd:
for line in fd:
- key, value = line.split(":")
+ try:
+ key, value = line.split(":")
+ except ValueError:
+ print (f"Unparsable line: '{line.strip()}' in {filename}", file=sys.stderr)
+ continue
key = key.strip()
value = int(value)
stats[key] += value
@@ -265,17 +270,20 @@ def emit_call_stats(stats):
def emit_object_stats(stats):
with Section("Object stats", summary="allocations, frees and dict materializatons"):
- total = stats.get("Object new values")
+ total_materializations = stats.get("Object new values")
+ total_allocations = stats.get("Object allocations")
rows = []
for key, value in stats.items():
if key.startswith("Object"):
if "materialize" in key:
- materialize = f"{100*value/total:0.1f}%"
+ ratio = f"{100*value/total_materializations:0.1f}%"
+ elif "allocations" in key:
+ ratio = f"{100*value/total_allocations:0.1f}%"
else:
- materialize = ""
+ ratio = ""
label = key[6:].strip()
label = label[0].upper() + label[1:]
- rows.append((label, value, materialize))
+ rows.append((label, value, ratio))
emit_table(("", "Count:", "Ratio:"), rows)
def get_total(opcode_stats):
More information about the Python-checkins
mailing list