Skip to content

Commit 2bde682

Browse files
brandtbucherbrandtbucher
authored
bpo-46841: Quicken code in-place (GH-31888)
* Moves the bytecode to the end of the corresponding PyCodeObject, and quickens it in-place. * Removes the almost-always-unused co_varnames, co_freevars, and co_cellvars member caches * _PyOpcode_Deopt is a new mapping from all opcodes to their un-quickened forms. * _PyOpcode_InlineCacheEntries is renamed to _PyOpcode_Caches * _Py_IncrementCountAndMaybeQuicken is renamed to _PyCode_Warmup * _Py_Quicken is renamed to _PyCode_Quicken * _co_quickened is renamed to _co_code_adaptive (and is now a read-only memoryview). * Do not emit unused nonzero opargs anymore in the compiler.
1 parent 08eb754 commit 2bde682

18 files changed

+831
-687
lines changed

‎Include/cpython/code.h

+74-83
Original file line numberDiff line numberDiff line change
@@ -26,91 +26,80 @@ typedef uint16_t _Py_CODEUNIT;
2626
// Use "unsigned char" instead of "uint8_t" here to avoid illegal aliasing:
2727
#define _Py_SET_OPCODE(word, opcode) (((unsigned char *)&(word))[0] = (opcode))
2828

29+
// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
30+
// defined in this macro:
31+
#define _PyCode_DEF(SIZE) { \
32+
PyObject_VAR_HEAD \
33+
\
34+
/* Note only the following fields are used in hash and/or comparisons \
35+
* \
36+
* - co_name \
37+
* - co_argcount \
38+
* - co_posonlyargcount \
39+
* - co_kwonlyargcount \
40+
* - co_nlocals \
41+
* - co_stacksize \
42+
* - co_flags \
43+
* - co_firstlineno \
44+
* - co_consts \
45+
* - co_names \
46+
* - co_localsplusnames \
47+
* This is done to preserve the name and line number for tracebacks \
48+
* and debuggers; otherwise, constant de-duplication would collapse \
49+
* identical functions/lambdas defined on different lines. \
50+
*/ \
51+
\
52+
/* These fields are set with provided values on new code objects. */ \
53+
\
54+
/* The hottest fields (in the eval loop) are grouped here at the top. */ \
55+
PyObject *co_consts; /* list (constants used) */ \
56+
PyObject *co_names; /* list of strings (names used) */ \
57+
PyObject *co_exceptiontable; /* Byte string encoding exception handling \
58+
table */ \
59+
int co_flags; /* CO_..., see below */ \
60+
int co_warmup; /* Warmup counter for quickening */ \
61+
\
62+
/* The rest are not so impactful on performance. */ \
63+
int co_argcount; /* #arguments, except *args */ \
64+
int co_posonlyargcount; /* #positional only arguments */ \
65+
int co_kwonlyargcount; /* #keyword only arguments */ \
66+
int co_stacksize; /* #entries needed for evaluation stack */ \
67+
int co_firstlineno; /* first source line number */ \
68+
\
69+
/* redundant values (derived from co_localsplusnames and \
70+
co_localspluskinds) */ \
71+
int co_nlocalsplus; /* number of local + cell + free variables \
72+
*/ \
73+
int co_nlocals; /* number of local variables */ \
74+
int co_nplaincellvars; /* number of non-arg cell variables */ \
75+
int co_ncellvars; /* total number of cell variables */ \
76+
int co_nfreevars; /* number of free variables */ \
77+
\
78+
PyObject *co_localsplusnames; /* tuple mapping offsets to names */ \
79+
PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte \
80+
per variable) */ \
81+
PyObject *co_filename; /* unicode (where it was loaded from) */ \
82+
PyObject *co_name; /* unicode (name, for reference) */ \
83+
PyObject *co_qualname; /* unicode (qualname, for reference) */ \
84+
PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) \
85+
See Objects/lnotab_notes.txt for details. \
86+
*/ \
87+
PyObject *co_endlinetable; /* bytes object that holds end lineno for \
88+
instructions separated across different \
89+
lines */ \
90+
PyObject *co_columntable; /* bytes object that holds start/end column \
91+
offset each instruction */ \
92+
\
93+
PyObject *co_weakreflist; /* to support weakrefs to code objects */ \
94+
/* Scratch space for extra data relating to the code object. \
95+
Type is a void* to keep the format private in codeobject.c to force \
96+
people to go through the proper APIs. */ \
97+
void *co_extra; \
98+
char co_code_adaptive[(SIZE)]; \
99+
}
29100

30101
/* Bytecode object */
31-
struct PyCodeObject {
32-
PyObject_HEAD
33-
34-
/* Note only the following fields are used in hash and/or comparisons
35-
*
36-
* - co_name
37-
* - co_argcount
38-
* - co_posonlyargcount
39-
* - co_kwonlyargcount
40-
* - co_nlocals
41-
* - co_stacksize
42-
* - co_flags
43-
* - co_firstlineno
44-
* - co_code
45-
* - co_consts
46-
* - co_names
47-
* - co_varnames
48-
* - co_freevars
49-
* - co_cellvars
50-
*
51-
* This is done to preserve the name and line number for tracebacks
52-
* and debuggers; otherwise, constant de-duplication would collapse
53-
* identical functions/lambdas defined on different lines.
54-
*/
55-
56-
/* These fields are set with provided values on new code objects. */
57-
58-
// The hottest fields (in the eval loop) are grouped here at the top.
59-
PyObject *co_consts; /* list (constants used) */
60-
PyObject *co_names; /* list of strings (names used) */
61-
_Py_CODEUNIT *co_firstinstr; /* Pointer to first instruction, used for quickening.
62-
Unlike the other "hot" fields, this one is
63-
actually derived from co_code. */
64-
PyObject *co_exceptiontable; /* Byte string encoding exception handling table */
65-
int co_flags; /* CO_..., see below */
66-
int co_warmup; /* Warmup counter for quickening */
67-
68-
// The rest are not so impactful on performance.
69-
int co_argcount; /* #arguments, except *args */
70-
int co_posonlyargcount; /* #positional only arguments */
71-
int co_kwonlyargcount; /* #keyword only arguments */
72-
int co_stacksize; /* #entries needed for evaluation stack */
73-
int co_firstlineno; /* first source line number */
74-
PyObject *co_code; /* instruction opcodes */
75-
PyObject *co_localsplusnames; /* tuple mapping offsets to names */
76-
PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte per variable) */
77-
PyObject *co_filename; /* unicode (where it was loaded from) */
78-
PyObject *co_name; /* unicode (name, for reference) */
79-
PyObject *co_qualname; /* unicode (qualname, for reference) */
80-
PyObject *co_linetable; /* bytes (encoding addr<->lineno mapping) See
81-
Objects/lnotab_notes.txt for details. */
82-
PyObject *co_endlinetable; /* bytes object that holds end lineno for
83-
instructions separated across different
84-
lines */
85-
PyObject *co_columntable; /* bytes object that holds start/end column
86-
offset each instruction */
87-
88-
/* These fields are set with computed values on new code objects. */
89-
90-
// redundant values (derived from co_localsplusnames and co_localspluskinds)
91-
int co_nlocalsplus; /* number of local + cell + free variables */
92-
int co_nlocals; /* number of local variables */
93-
int co_nplaincellvars; /* number of non-arg cell variables */
94-
int co_ncellvars; /* total number of cell variables */
95-
int co_nfreevars; /* number of free variables */
96-
// lazily-computed values
97-
PyObject *co_varnames; /* tuple of strings (local variable names) */
98-
PyObject *co_cellvars; /* tuple of strings (cell variable names) */
99-
PyObject *co_freevars; /* tuple of strings (free variable names) */
100-
101-
/* The remaining fields are zeroed out on new code objects. */
102-
103-
PyObject *co_weakreflist; /* to support weakrefs to code objects */
104-
/* Scratch space for extra data relating to the code object.
105-
Type is a void* to keep the format private in codeobject.c to force
106-
people to go through the proper APIs. */
107-
void *co_extra;
108-
/* Quickened instructions and cache, or NULL
109-
This should be treated as opaque by all code except the specializer and
110-
interpreter. */
111-
_Py_CODEUNIT *co_quickened;
112-
113-
};
102+
struct PyCodeObject _PyCode_DEF(1);
114103

115104
/* Masks for co_flags above */
116105
#define CO_OPTIMIZED 0x0001
@@ -151,6 +140,8 @@ PyAPI_DATA(PyTypeObject) PyCode_Type;
151140

152141
#define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
153142
#define PyCode_GetNumFree(op) ((op)->co_nfreevars)
143+
#define _PyCode_CODE(CO) ((_Py_CODEUNIT *)(CO)->co_code_adaptive)
144+
#define _PyCode_NBYTES(CO) (Py_SIZE(CO) * (Py_ssize_t)sizeof(_Py_CODEUNIT))
154145

155146
/* Public interface */
156147
PyAPI_FUNC(PyCodeObject *) PyCode_New(

‎Include/internal/pycore_code.h

+5-12
Original file line numberDiff line numberDiff line change
@@ -92,30 +92,22 @@ typedef struct {
9292

9393
#define INLINE_CACHE_ENTRIES_STORE_SUBSCR CACHE_ENTRIES(_PyStoreSubscrCache)
9494

95-
/* Maximum size of code to quicken, in code units. */
96-
#define MAX_SIZE_TO_QUICKEN 10000
97-
9895
#define QUICKENING_WARMUP_DELAY 8
9996

10097
/* We want to compare to zero for efficiency, so we offset values accordingly */
10198
#define QUICKENING_INITIAL_WARMUP_VALUE (-QUICKENING_WARMUP_DELAY)
102-
#define QUICKENING_WARMUP_COLDEST 1
10399

104-
int _Py_Quicken(PyCodeObject *code);
100+
void _PyCode_Quicken(PyCodeObject *code);
105101

106-
/* Returns 1 if quickening occurs.
107-
* -1 if an error occurs
108-
* 0 otherwise */
109-
static inline int
110-
_Py_IncrementCountAndMaybeQuicken(PyCodeObject *code)
102+
static inline void
103+
_PyCode_Warmup(PyCodeObject *code)
111104
{
112105
if (code->co_warmup != 0) {
113106
code->co_warmup++;
114107
if (code->co_warmup == 0) {
115-
return _Py_Quicken(code) ? -1 : 1;
108+
_PyCode_Quicken(code);
116109
}
117110
}
118-
return 0;
119111
}
120112

121113
extern Py_ssize_t _Py_QuickenedCount;
@@ -225,6 +217,7 @@ PyAPI_FUNC(PyCodeObject *) _PyCode_New(struct _PyCodeConstructor *);
225217
extern PyObject* _PyCode_GetVarnames(PyCodeObject *);
226218
extern PyObject* _PyCode_GetCellvars(PyCodeObject *);
227219
extern PyObject* _PyCode_GetFreevars(PyCodeObject *);
220+
extern PyObject* _PyCode_GetCode(PyCodeObject *);
228221

229222
/* Return the ending source code line number from a bytecode index. */
230223
extern int _PyCode_Addr2EndLine(PyCodeObject *, int);

0 commit comments

Comments
 (0)