GH-96458: Statically initialize utf8 representation of static strings (#96481)

Author: kumaraditya303

Include/internal/pycore_runtime_init.h

@@ -113,10 +113,12 @@ extern "C" {
     ._ ## NAME = _PyASCIIObject_INIT(LITERAL)
 #define INIT_ID(NAME) \
     ._ ## NAME = _PyASCIIObject_INIT(#NAME)
-#define _PyUnicode_LATIN1_INIT(LITERAL) \
+#define _PyUnicode_LATIN1_INIT(LITERAL, UTF8) \
     { \
         ._latin1 = { \
             ._base = _PyUnicode_ASCII_BASE_INIT((LITERAL), 0), \
+            .utf8 = (UTF8), \
+            .utf8_length = sizeof(UTF8) - 1, \
         }, \
         ._data = (LITERAL), \
     }

Include/internal/pycore_runtime_init_generated.h

@@ -19,7 +19,6 @@ extern PyStatus _PyUnicode_InitGlobalObjects(PyInterpreterState *);
 extern PyStatus _PyUnicode_InitTypes(PyInterpreterState *);
 extern void _PyUnicode_Fini(PyInterpreterState *);
 extern void _PyUnicode_FiniTypes(PyInterpreterState *);
-extern void _PyStaticUnicode_Dealloc(PyObject *);
 
 extern PyTypeObject _PyUnicodeASCIIIter_Type;
 

Include/internal/pycore_unicodeobject.h

@@ -15184,23 +15184,6 @@ _PyUnicode_FiniTypes(PyInterpreterState *interp)
 }
 
 
-static void unicode_static_dealloc(PyObject *op)
-{
-    PyASCIIObject *ascii = _PyASCIIObject_CAST(op);
-
-    assert(ascii->state.compact);
-
-    if (!ascii->state.ascii) {
-        PyCompactUnicodeObject* compact = (PyCompactUnicodeObject*)op;
-        if (compact->utf8) {
-            PyObject_Free(compact->utf8);
-            compact->utf8 = NULL;
-            compact->utf8_length = 0;
-        }
-    }
-}
-
-
 void
 _PyUnicode_Fini(PyInterpreterState *interp)
 {
@@ -15217,24 +15200,8 @@ _PyUnicode_Fini(PyInterpreterState *interp)
     _PyUnicode_FiniEncodings(&state->fs_codec);
 
     unicode_clear_identifiers(state);
-
-    // Clear the single character singletons
-    for (int i = 0; i < 128; i++) {
-        unicode_static_dealloc((PyObject*)&_Py_SINGLETON(strings).ascii[i]);
-    }
-    for (int i = 0; i < 128; i++) {
-        unicode_static_dealloc((PyObject*)&_Py_SINGLETON(strings).latin1[i]);
-    }
 }
 
-
-void
-_PyStaticUnicode_Dealloc(PyObject *op)
-{
-    unicode_static_dealloc(op);
-}
-
-
 /* A _string module, to export formatter_parser and formatter_field_name_split
    to the string.Formatter class implemented in Python. */
 

Objects/unicodeobject.c

@@ -195,7 +195,6 @@ def generate_unicode(self, name: str, s: str) -> str:
                 else:
                     self.write("PyCompactUnicodeObject _compact;")
                 self.write(f"{datatype} _data[{len(s)+1}];")
-        self.deallocs.append(f"_PyStaticUnicode_Dealloc((PyObject *)&{name});")
         with self.block(f"{name} =", ";"):
             if ascii:
                 with self.block("._ascii =", ","):
@@ -218,6 +217,9 @@ def generate_unicode(self, name: str, s: str) -> str:
                             self.write(f".kind = {kind},")
                             self.write(".compact = 1,")
                             self.write(".ascii = 0,")
+                    utf8 = s.encode('utf-8')
+                    self.write(f'.utf8 = {make_string_literal(utf8)},')
+                    self.write(f'.utf8_length = {len(utf8)},')
                 with self.block(f"._data =", ","):
                     for i in range(0, len(s), 16):
                         data = s[i:i+16]

Tools/scripts/deepfreeze.py

@@ -287,7 +287,11 @@ def generate_runtime_init(identifiers, strings):
                             immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(strings).ascii[{i}]')
                     with printer.block('.latin1 =', ','):
                         for i in range(128, 256):
-                            printer.write(f'_PyUnicode_LATIN1_INIT("\\x{i:02x}"),')
+                            utf8 = ['"']
+                            for c in chr(i).encode('utf-8'):
+                                utf8.append(f"\\x{c:02x}")
+                            utf8.append('"')
+                            printer.write(f'_PyUnicode_LATIN1_INIT("\\x{i:02x}", {"".join(utf8)}),')
                             immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(strings).latin1[{i} - 128]')
                 printer.write('')
                 with printer.block('.tuple_empty =', ','):