Issue python#28618: Make hot functions using __attribute__((hot))

vstinner · vstinner · commit c6944e7edcac · 2016-11-11T02:13:35.000+01:00
When Python is not compiled with PGO, the performance of Python on call_simple
and call_method microbenchmarks depend highly on the code placement. In the
worst case, the performance slowdown can be up to 70%.

The GCC __attribute__((hot)) attribute helps to keep hot code close to reduce
the risk of such major slowdown. This attribute is ignored when Python is
compiled with PGO.

The following functions are considered as hot according to statistics collected
by perf record/perf report:

* _PyEval_EvalFrameDefault()
* call_function()
* _PyFunction_FastCall()
* PyFrame_New()
* frame_dealloc()
* PyErr_Occurred()
diff --git a/Include/pyport.h b/Include/pyport.h
@@ -490,13 +490,36 @@ extern "C" {
  *    typedef int T1 Py_DEPRECATED(2.4);
  *    extern int x() Py_DEPRECATED(2.5);
  */
-#if defined(__GNUC__) && ((__GNUC__ >= 4) || \
-              (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))
+#if defined(__GNUC__) \
+    && ((__GNUC__ >= 4) || (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1))
 #define Py_DEPRECATED(VERSION_UNUSED) __attribute__((__deprecated__))
 #else
 #define Py_DEPRECATED(VERSION_UNUSED)
 #endif
 
+
+/* Py_HOT_FUNCTION
+ * The hot attribute on a function is used to inform the compiler that the
+ * function is a hot spot of the compiled program. The function is optimized
+ * more aggressively and on many target it is placed into special subsection of
+ * the text section so all hot functions appears close together improving
+ * locality.
+ *
+ * Usage:
+ *    int Py_HOT_FUNCTION x() { return 3; }
+ *
+ * Issue #28618: This attribute must not be abused, otherwise it can have a
+ * negative effect on performance. Only the functions were Python spend most of
+ * its time must use it. Use a profiler when running performance benchmark
+ * suite to find these functions.
+ */
+#if defined(__GNUC__) \
+    && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 3))
+#define _Py_HOT_FUNCTION __attribute__((hot))
+#else
+#define _Py_HOT_FUNCTION
+#endif
+
 /**************************************************************************
 Prototypes that are missing from the standard include files on some systems
 (and possibly only some versions of such systems.)
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
@@ -409,7 +409,7 @@ static int numfree = 0;         /* number of frames currently in free_list */
 /* max value for numfree */
 #define PyFrame_MAXFREELIST 200
 
-static void
+static void _Py_HOT_FUNCTION
 frame_dealloc(PyFrameObject *f)
 {
     PyObject **p, **valuestack;
@@ -605,7 +605,7 @@ int _PyFrame_Init()
     return 1;
 }
 
-PyFrameObject *
+PyFrameObject* _Py_HOT_FUNCTION
 PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
             PyObject *locals)
 {
diff --git a/Python/ceval.c b/Python/ceval.c
@@ -718,7 +718,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
     return tstate->interp->eval_frame(f, throwflag);
 }
 
-PyObject *
+PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
 {
 #ifdef DXPAIRS
@@ -4771,7 +4771,7 @@ if (tstate->use_tracing && tstate->c_profilefunc) { \
     x = call; \
     }
 
-static PyObject *
+static PyObject* _Py_HOT_FUNCTION
 call_function(PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames)
 {
     PyObject **pfunc = (*pp_stack) - oparg - 1;
@@ -4844,7 +4844,7 @@ call_function(PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames)
    done before evaluating the frame.
 */
 
-static PyObject*
+static PyObject* _Py_HOT_FUNCTION
 _PyFunction_FastCall(PyCodeObject *co, PyObject **args, Py_ssize_t nargs,
                      PyObject *globals)
 {
diff --git a/Python/errors.c b/Python/errors.c
@@ -158,7 +158,7 @@ PyErr_SetString(PyObject *exception, const char *string)
 }
 
 
-PyObject *
+PyObject* _Py_HOT_FUNCTION
 PyErr_Occurred(void)
 {
     PyThreadState *tstate = PyThreadState_GET();

Original file line number	Diff line number	Diff line change
`@@ -409,7 +409,7 @@ static int numfree = 0; /* number of frames currently in free_list */`
`409`	`409`	`/* max value for numfree */`
`410`	`410`	`#define PyFrame_MAXFREELIST 200`
`411`	`411`
`412`		`-static void`
	`412`	`+static void _Py_HOT_FUNCTION`
`413`	`413`	`frame_dealloc(PyFrameObject *f)`
`414`	`414`	`{`
`415`	`415`	`PyObject p, valuestack;`
`@@ -605,7 +605,7 @@ int _PyFrame_Init()`
`605`	`605`	`return 1;`
`606`	`606`	`}`
`607`	`607`
`608`		`-PyFrameObject *`
	`608`	`+PyFrameObject* _Py_HOT_FUNCTION`
`609`	`609`	`PyFrame_New(PyThreadState tstate, PyCodeObject code, PyObject *globals,`
`610`	`610`	`PyObject *locals)`
`611`	`611`	`{`
Original file line number	Diff line number	Diff line change
`@@ -718,7 +718,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)`
`718`	`718`	`return tstate->interp->eval_frame(f, throwflag);`
`719`	`719`	`}`
`720`	`720`
`721`		`-PyObject *`
	`721`	`+PyObject* _Py_HOT_FUNCTION`
`722`	`722`	`_PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)`
`723`	`723`	`{`
`724`	`724`	`#ifdef DXPAIRS`
`@@ -4771,7 +4771,7 @@ if (tstate->use_tracing && tstate->c_profilefunc) { \`
`4771`	`4771`	`x = call; \`
`4772`	`4772`	`}`
`4773`	`4773`
`4774`		`-static PyObject *`
	`4774`	`+static PyObject* _Py_HOT_FUNCTION`
`4775`	`4775`	`call_function(PyObject **pp_stack, Py_ssize_t oparg, PyObject kwnames)`
`4776`	`4776`	`{`
`4777`	`4777`	`PyObject *pfunc = (pp_stack) - oparg - 1;`
`@@ -4844,7 +4844,7 @@ call_function(PyObject **pp_stack, Py_ssize_t oparg, PyObject kwnames)`
`4844`	`4844`	`done before evaluating the frame.`
`4845`	`4845`	`*/`
`4846`	`4846`
`4847`		`-static PyObject*`
	`4847`	`+static PyObject* _Py_HOT_FUNCTION`
`4848`	`4848`	`_PyFunction_FastCall(PyCodeObject co, PyObject *args, Py_ssize_t nargs,`
`4849`	`4849`	`PyObject *globals)`
`4850`	`4850`	`{`
Original file line number	Diff line number	Diff line change
`@@ -158,7 +158,7 @@ PyErr_SetString(PyObject exception, const char string)`
`158`	`158`	`}`
`159`	`159`
`160`	`160`
`161`		`-PyObject *`
	`161`	`+PyObject* _Py_HOT_FUNCTION`
`162`	`162`	`PyErr_Occurred(void)`
`163`	`163`	`{`
`164`	`164`	`PyThreadState *tstate = PyThreadState_GET();`