bpo-44337: Port LOAD_ATTR to PEP 659 adaptive interpreter (GH-26595)

* Specialize LOAD_ATTR with  LOAD_ATTR_SLOT and LOAD_ATTR_SPLIT_KEYS

* Move dict-common.h to internal/pycore_dict.h

* Add LOAD_ATTR_WITH_HINT specialized opcode.

* Quicken in function if loopy

* Specialize LOAD_ATTR for module attributes.

* Add specialization stats

Author: markshannon

Include/internal/pycore_code.h

@@ -42,6 +42,12 @@ typedef struct {
     uint16_t index;
 } _PyAdaptiveEntry;
 
+
+typedef struct {
+    uint32_t tp_version;
+    uint32_t dk_version_or_hint;
+} _PyLoadAttrCache;
+
 /* Add specialized versions of entries to this union.
  *
  * Do not break the invariant: sizeof(SpecializedCacheEntry) == 8
@@ -55,6 +61,7 @@ typedef struct {
 typedef union {
     _PyEntryZero zero;
     _PyAdaptiveEntry adaptive;
+    _PyLoadAttrCache load_attr;
 } SpecializedCacheEntry;
 
 #define INSTRUCTIONS_PER_ENTRY (sizeof(SpecializedCacheEntry)/sizeof(_Py_CODEUNIT))
@@ -255,6 +262,83 @@ PyAPI_FUNC(PyObject *) _PyCode_GetCellvars(PyCodeObject *);
 PyAPI_FUNC(PyObject *) _PyCode_GetFreevars(PyCodeObject *);
 
 
+/* Cache hits and misses */
+
+static inline uint8_t
+saturating_increment(uint8_t c)
+{
+    return c<<1;
+}
+
+static inline uint8_t
+saturating_decrement(uint8_t c)
+{
+    return (c>>1) + 128;
+}
+
+static inline uint8_t
+saturating_zero(void)
+{
+    return 255;
+}
+
+/* Starting value for saturating counter.
+ * Technically this should be 1, but that is likely to
+ * cause a bit of thrashing when we optimize then get an immediate miss.
+ * We want to give the counter a change to stabilize, so we start at 3.
+ */
+static inline uint8_t
+saturating_start(void)
+{
+    return saturating_zero()<<3;
+}
+
+static inline void
+record_cache_hit(_PyAdaptiveEntry *entry) {
+    entry->counter = saturating_increment(entry->counter);
+}
+
+static inline void
+record_cache_miss(_PyAdaptiveEntry *entry) {
+    entry->counter = saturating_decrement(entry->counter);
+}
+
+static inline int
+too_many_cache_misses(_PyAdaptiveEntry *entry) {
+    return entry->counter == saturating_zero();
+}
+
+#define BACKOFF 64
+
+static inline void
+cache_backoff(_PyAdaptiveEntry *entry) {
+    entry->counter = BACKOFF;
+}
+
+/* Specialization functions */
+
+int _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, SpecializedCacheEntry *cache);
+
+#define SPECIALIZATION_STATS 0
+#if SPECIALIZATION_STATS
+
+typedef struct _specialization_stats {
+    uint64_t specialization_success;
+    uint64_t specialization_failure;
+    uint64_t loadattr_hit;
+    uint64_t loadattr_deferred;
+    uint64_t loadattr_miss;
+    uint64_t loadattr_deopt;
+} SpecializationStats;
+
+extern SpecializationStats _specialization_stats;
+#define STAT_INC(name) _specialization_stats.name++
+void _Py_PrintSpecializationStats(void);
+#else
+#define STAT_INC(name) ((void)0)
+#endif
+
+
 #ifdef __cplusplus
 }
 #endif

Objects/dict-common.h renamed to Include/internal/pycore_dict.h

@@ -1,5 +1,14 @@
-#ifndef Py_DICT_COMMON_H
-#define Py_DICT_COMMON_H
+
+#ifndef Py_INTERNAL_DICT_H
+#define Py_INTERNAL_DICT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
 
 typedef struct {
     /* Cached hash code of me_key. */
@@ -62,4 +71,26 @@ struct _dictkeysobject {
        see the DK_ENTRIES() macro */
 };
 
+#define DK_LOG_SIZE(dk)  ((dk)->dk_log2_size)
+#if SIZEOF_VOID_P > 4
+#define DK_SIZE(dk)      (((int64_t)1)<<DK_LOG_SIZE(dk))
+#define DK_IXSIZE(dk)                     \
+    (DK_LOG_SIZE(dk) <= 7 ?               \
+        1 : DK_LOG_SIZE(dk) <= 15 ?       \
+            2 : DK_LOG_SIZE(dk) <= 31 ?   \
+                4 : sizeof(int64_t))
+#else
+#define DK_SIZE(dk)      (1<<DK_LOG_SIZE(dk))
+#define DK_IXSIZE(dk)                     \
+    (DK_LOG_SIZE(dk) <= 7 ?               \
+        1 : DK_LOG_SIZE(dk) <= 15 ?       \
+            2 : sizeof(int32_t))
+#endif
+#define DK_ENTRIES(dk) \
+    ((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
+
+
+#ifdef __cplusplus
+}
 #endif
+#endif   /* !Py_INTERNAL_DICT_H */

Include/opcode.h

@@ -218,3 +218,12 @@ def jabs_op(name, op):
 def_op('CALL_METHOD_KW', 166)
 
 del def_op, name_op, jrel_op, jabs_op
+
+_specialized_instructions = [
+    "JUMP_ABSOLUTE_QUICK",
+    "LOAD_ATTR_ADAPTIVE",
+    "LOAD_ATTR_SPLIT_KEYS",
+    "LOAD_ATTR_WITH_HINT",
+    "LOAD_ATTR_SLOT",
+    "LOAD_ATTR_MODULE",
+]

Lib/opcode.py

@@ -323,7 +323,7 @@ class C(): pass
                         break
         """
         rc, out, err = assert_python_ok('-c', code)
-        self.assertIn(b'MemoryError 1 10', out)
+        self.assertIn(b'MemoryError 1', out)
         self.assertIn(b'MemoryError 2 20', out)
         self.assertIn(b'MemoryError 3 30', out)
 

Lib/test/test_capi.py

@@ -979,8 +979,7 @@ Objects/bytearrayobject.o: $(srcdir)/Objects/bytearrayobject.c $(BYTESTR_DEPS)
 
 Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS)
 
-Objects/odictobject.o: $(srcdir)/Objects/dict-common.h
-Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h $(srcdir)/Objects/dict-common.h
+Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h
 Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
 
 .PHONY: regen-opcode-targets
@@ -1156,6 +1155,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_compile.h \
 		$(srcdir)/Include/internal/pycore_condvar.h \
 		$(srcdir)/Include/internal/pycore_context.h \
+		$(srcdir)/Include/internal/pycore_dict.h \
 		$(srcdir)/Include/internal/pycore_dtoa.h \
 		$(srcdir)/Include/internal/pycore_fileutils.h \
 		$(srcdir)/Include/internal/pycore_format.h \

Makefile.pre.in

@@ -0,0 +1,11 @@
+Initial implementation of adaptive specialization of LOAD_ATTR
+
+Four specialized forms of LOAD_ATTR are added:
+
+* LOAD_ATTR_SLOT
+
+* LOAD_ATTR_SPLIT_KEYS
+
+* LOAD_ATTR_WITH_HINT
+
+* LOAD_ATTR_MODULE

Misc/NEWS.d/next/Core and Builtins/2021-06-08-10-22-46.bpo-44337.RTjmIt.rst

@@ -117,7 +117,7 @@ converting the dict to the combined table.
 #include "pycore_object.h"   // _PyObject_GC_TRACK()
 #include "pycore_pyerrors.h" // _PyErr_Fetch()
 #include "pycore_pystate.h"  // _PyThreadState_GET()
-#include "dict-common.h"
+#include "pycore_dict.h"
 #include "stringlib/eq.h"    // unicode_eq()
 
 /*[clinic input]
@@ -285,24 +285,6 @@ _PyDict_DebugMallocStats(FILE *out)
                            state->numfree, sizeof(PyDictObject));
 }
 
-#define DK_LOG_SIZE(dk)  ((dk)->dk_log2_size)
-#if SIZEOF_VOID_P > 4
-#define DK_SIZE(dk)      (((int64_t)1)<<DK_LOG_SIZE(dk))
-#define DK_IXSIZE(dk)                     \
-    (DK_LOG_SIZE(dk) <= 7 ?               \
-        1 : DK_LOG_SIZE(dk) <= 15 ?       \
-            2 : DK_LOG_SIZE(dk) <= 31 ?   \
-                4 : sizeof(int64_t))
-#else
-#define DK_SIZE(dk)      (1<<DK_LOG_SIZE(dk))
-#define DK_IXSIZE(dk)                     \
-    (DK_LOG_SIZE(dk) <= 7 ?               \
-        1 : DK_LOG_SIZE(dk) <= 15 ?       \
-            2 : sizeof(int32_t))
-#endif
-#define DK_ENTRIES(dk) \
-    ((PyDictKeyEntry*)(&((int8_t*)((dk)->dk_indices))[DK_SIZE(dk) * DK_IXSIZE(dk)]))
-
 #define DK_MASK(dk) (DK_SIZE(dk)-1)
 #define IS_POWER_OF_2(x) (((x) & (x-1)) == 0)
 
@@ -1544,10 +1526,10 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
     assert(hashpos >= 0);
 
     mp->ma_used--;
+    mp->ma_keys->dk_version = 0;
     mp->ma_version_tag = DICT_NEXT_VERSION();
     ep = &DK_ENTRIES(mp->ma_keys)[ix];
     dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY);
-    mp->ma_keys->dk_version = 0;
     old_key = ep->me_key;
     ep->me_key = NULL;
     ep->me_value = NULL;

Objects/dictobject.c

@@ -467,7 +467,7 @@ Potential Optimizations
 #include "Python.h"
 #include "pycore_object.h"
 #include <stddef.h>               // offsetof()
-#include "dict-common.h"
+#include "pycore_dict.h"
 #include <stddef.h>
 
 #include "clinic/odictobject.c.h"