diff --git a/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst
new file mode 100644
index 00000000000000..a84e1feb750251
--- /dev/null
+++ b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst
@@ -0,0 +1,2 @@
+Implement changes to build with deep-frozen modules on Windows.
+Note that we now require Python 3.10 as the "bootstrap" or "host" Python.
diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj
index d33e07c54b8c9c..288302f8f93579 100644
--- a/PCbuild/_freeze_module.vcxproj
+++ b/PCbuild/_freeze_module.vcxproj
@@ -234,101 +234,141 @@
importlib._bootstrap
$(IntDir)importlib._bootstrap.g.h
$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h
+ $(IntDir)importlib._bootstrap.g.c
+ $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap.c
importlib._bootstrap_external
$(IntDir)importlib._bootstrap_external.g.h
$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h
+ $(IntDir)importlib._bootstrap_external.g.c
+ $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap_external.c
zipimport
$(IntDir)zipimport.g.h
$(PySourcePath)Python\frozen_modules\zipimport.h
+ $(IntDir)zipimport.g.c
+ $(PySourcePath)Python\deepfreeze\df.zipimport.c
abc
$(IntDir)abc.g.h
$(PySourcePath)Python\frozen_modules\abc.h
+ $(IntDir)abc.g.c
+ $(PySourcePath)Python\deepfreeze\df.abc.c
codecs
$(IntDir)codecs.g.h
$(PySourcePath)Python\frozen_modules\codecs.h
+ $(IntDir)codecs.g.c
+ $(PySourcePath)Python\deepfreeze\df.codecs.c
io
$(IntDir)io.g.h
$(PySourcePath)Python\frozen_modules\io.h
+ $(IntDir)io.g.c
+ $(PySourcePath)Python\deepfreeze\df.io.c
_collections_abc
$(IntDir)_collections_abc.g.h
$(PySourcePath)Python\frozen_modules\_collections_abc.h
+ $(IntDir)_collections_abc.g.c
+ $(PySourcePath)Python\deepfreeze\df._collections_abc.c
_sitebuiltins
$(IntDir)_sitebuiltins.g.h
$(PySourcePath)Python\frozen_modules\_sitebuiltins.h
+ $(IntDir)_sitebuiltins.g.c
+ $(PySourcePath)Python\deepfreeze\df._sitebuiltins.c
genericpath
$(IntDir)genericpath.g.h
$(PySourcePath)Python\frozen_modules\genericpath.h
+ $(IntDir)genericpath.g.c
+ $(PySourcePath)Python\deepfreeze\df.genericpath.c
ntpath
$(IntDir)ntpath.g.h
$(PySourcePath)Python\frozen_modules\ntpath.h
+ $(IntDir)ntpath.g.c
+ $(PySourcePath)Python\deepfreeze\df.ntpath.c
posixpath
$(IntDir)posixpath.g.h
$(PySourcePath)Python\frozen_modules\posixpath.h
+ $(IntDir)posixpath.g.c
+ $(PySourcePath)Python\deepfreeze\df.posixpath.c
os
$(IntDir)os.g.h
$(PySourcePath)Python\frozen_modules\os.h
+ $(IntDir)os.g.c
+ $(PySourcePath)Python\deepfreeze\df.os.c
site
$(IntDir)site.g.h
$(PySourcePath)Python\frozen_modules\site.h
+ $(IntDir)site.g.c
+ $(PySourcePath)Python\deepfreeze\df.site.c
stat
$(IntDir)stat.g.h
$(PySourcePath)Python\frozen_modules\stat.h
+ $(IntDir)stat.g.c
+ $(PySourcePath)Python\deepfreeze\df.stat.c
__hello__
$(IntDir)__hello__.g.h
$(PySourcePath)Python\frozen_modules\__hello__.h
+ $(IntDir)__hello__.g.c
+ $(PySourcePath)Python\deepfreeze\df.__hello__.c
__phello__
$(IntDir)__phello__.g.h
$(PySourcePath)Python\frozen_modules\__phello__.h
+ $(IntDir)__phello__.g.c
+ $(PySourcePath)Python\deepfreeze\df.__phello__.c
__phello__.ham
$(IntDir)__phello__.ham.g.h
$(PySourcePath)Python\frozen_modules\__phello__.ham.h
+ $(IntDir)__phello__.ham.g.c
+ $(PySourcePath)Python\deepfreeze\df.__phello__.ham.c
__phello__.ham.eggs
$(IntDir)__phello__.ham.eggs.g.h
$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h
+ $(IntDir)__phello__.ham.eggs.g.c
+ $(PySourcePath)Python\deepfreeze\df.__phello__.ham.eggs.c
__phello__.spam
$(IntDir)__phello__.spam.g.h
$(PySourcePath)Python\frozen_modules\__phello__.spam.h
+ $(IntDir)__phello__.spam.g.c
+ $(PySourcePath)Python\deepfreeze\df.__phello__.spam.c
frozen_only
$(IntDir)frozen_only.g.h
$(PySourcePath)Python\frozen_modules\frozen_only.h
+ $(IntDir)frozen_only.g.c
+ $(PySourcePath)Python\deepfreeze\df.frozen_only.c
@@ -336,17 +376,29 @@
-
+
-
+ Condition="!Exists(%(None.OutFile)) or (Exists(%(None.DeepIntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(`
`, `
`))' != '$([System.IO.File]::ReadAllText(%(None.DeepIntFile)).Replace(`
`, `
`))')">
+
+
+
+
+
+
+
+
+
diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat
index d0e4a86b01fd03..a9f14c5277ffa0 100644
--- a/PCbuild/find_python.bat
+++ b/PCbuild/find_python.bat
@@ -31,13 +31,13 @@
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
@rem If we have Python in externals, use that one
-@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86"
@rem If HOST_PYTHON is recent enough, use that
-@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
@rem If py.exe finds a recent enough version, use that one
-@for %%p in (3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
+@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
@set _Py_NUGET=%NUGET%
diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj
index b65998186927b6..ece15606b5ed70 100644
--- a/PCbuild/pythoncore.vcxproj
+++ b/PCbuild/pythoncore.vcxproj
@@ -500,6 +500,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/Python/frozen.c b/Python/frozen.c
index 1565c9a3d73f7b..9f43db70886f79 100644
--- a/Python/frozen.c
+++ b/Python/frozen.c
@@ -61,12 +61,7 @@
#include "frozen_modules/frozen_only.h"
/* End includes */
-#ifdef MS_WINDOWS
-/* Deepfreeze isn't supported on Windows yet. */
-#define GET_CODE(name) NULL
-#else
#define GET_CODE(name) _Py_get_##name##_toplevel
-#endif
/* Start extern declarations */
extern PyObject *_Py_get_importlib__bootstrap_toplevel(void);
diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py
index 074127f9492988..b6d52b7454965f 100644
--- a/Tools/scripts/deepfreeze.py
+++ b/Tools/scripts/deepfreeze.py
@@ -1,13 +1,16 @@
import argparse
+import ast
import builtins
import collections
import contextlib
import os
-import sys
+import re
import time
import types
import typing
+import umarshal
+
verbose = False
@@ -55,7 +58,8 @@ def get_localsplus_counts(code: types.CodeType,
nplaincellvars += 1
elif kind & CO_FAST_FREE:
nfreevars += 1
- assert nlocals == len(code.co_varnames) == code.co_nlocals
+ assert nlocals == len(code.co_varnames) == code.co_nlocals, \
+ (nlocals, len(code.co_varnames), code.co_nlocals)
assert ncellvars == len(code.co_cellvars)
assert nfreevars == len(code.co_freevars)
assert len(names) == nlocals + nplaincellvars + nfreevars
@@ -274,14 +278,7 @@ def generate_tuple(self, name: str, t: tuple[object, ...]) -> str:
self.write(item + ",")
return f"& {name}._object.ob_base.ob_base"
- def generate_int(self, name: str, i: int) -> str:
- maxint = sys.maxsize
- if maxint == 2**31 - 1:
- digit = 2**15
- elif maxint == 2**63 - 1:
- digit = 2**30
- else:
- assert False, f"What int size is this system?!? {maxint=}"
+ def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None:
sign = -1 if i < 0 else 0 if i == 0 else +1
i = abs(i)
digits: list[int] = []
@@ -298,6 +295,20 @@ def generate_int(self, name: str, i: int) -> str:
if digits:
ds = ", ".join(map(str, digits))
self.write(f".ob_digit = {{ {ds} }},")
+
+ def generate_int(self, name: str, i: int) -> str:
+ if abs(i) < 2**15:
+ self._generate_int_for_bits(name, i, 2**15)
+ else:
+ connective = "if"
+ for bits_in_digit in 15, 30:
+ self.write(f"#{connective} PYLONG_BITS_IN_DIGIT == {bits_in_digit}")
+ self._generate_int_for_bits(name, i, 2**bits_in_digit)
+ connective = "elif"
+ self.write("#else")
+ self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"')
+ self.write("#endif")
+ # If neither clause applies, it won't compile
return f"& {name}.ob_base.ob_base"
def generate_float(self, name: str, x: float) -> str:
@@ -326,7 +337,7 @@ def generate(self, name: str, obj: object) -> str:
return self.cache[key]
self.misses += 1
match obj:
- case types.CodeType() as code:
+ case types.CodeType() | umarshal.Code() as code:
val = self.generate_code(name, code)
case tuple(t):
val = self.generate_tuple(name, t)
@@ -367,8 +378,31 @@ def generate(self, name: str, obj: object) -> str:
}
"""
+FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */"
+
+FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*"
+
+
+def is_frozen_header(source: str) -> bool:
+ return source.startswith(FROZEN_COMMENT)
+
+
+def decode_frozen_data(source: str) -> types.CodeType:
+ lines = source.splitlines()
+ while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None:
+ del lines[0]
+ while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None:
+ del lines[-1]
+ values: tuple[int, ...] = ast.literal_eval("".join(lines))
+ data = bytes(values)
+ return umarshal.loads(data)
+
+
def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None:
- code = compile(source, filename, "exec")
+ if is_frozen_header(source):
+ code = decode_frozen_data(source)
+ else:
+ code = compile(source, filename, "exec")
printer = Printer(file)
printer.generate("toplevel", code)
printer.write("")
diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index ccea4e11ab6ca3..61ccae61e4f965 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -11,7 +11,6 @@
import platform
import subprocess
import sys
-import textwrap
import time
from update_file import updating_file_with_tmpfile, update_file_with_tmpfile
@@ -55,6 +54,7 @@ def find_tool():
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters')
+PCBUILD_PYTHONCORE = os.path.join(ROOT_DIR, 'PCbuild', 'pythoncore.vcxproj')
OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath'
@@ -717,20 +717,28 @@ def regen_makefile(modules):
def regen_pcbuild(modules):
projlines = []
filterlines = []
+ corelines = []
for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
+ deepbase = "df." + src.id
+ deepoutfile = f"Python\\deepfreeze\\{deepbase}.c"
intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h'
+ deepintfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.c'
projlines.append(f' ')
projlines.append(f' {src.frozenid}')
projlines.append(f' $(IntDir){intfile}')
projlines.append(f' $(PySourcePath){header}')
+ projlines.append(f' $(IntDir){deepintfile}')
+ projlines.append(f' $(PySourcePath){deepoutfile}')
projlines.append(f' ')
filterlines.append(f' ')
filterlines.append(' Python Files')
filterlines.append(' ')
+ corelines.append(f' ')
+
print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
@@ -753,6 +761,17 @@ def regen_pcbuild(modules):
PCBUILD_FILTERS,
)
outfile.writelines(lines)
+ print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}')
+ with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile):
+ lines = infile.readlines()
+ lines = replace_block(
+ lines,
+ '',
+ '',
+ corelines,
+ PCBUILD_FILTERS,
+ )
+ outfile.writelines(lines)
#######################################
diff --git a/Tools/scripts/startuptime.py b/Tools/scripts/startuptime.py
new file mode 100644
index 00000000000000..1bb5b208f66e04
--- /dev/null
+++ b/Tools/scripts/startuptime.py
@@ -0,0 +1,22 @@
+# Quick script to time startup for various binaries
+
+import subprocess
+import sys
+import time
+
+NREPS = 100
+
+
+def main():
+ binaries = sys.argv[1:]
+ for bin in binaries:
+ t0 = time.time()
+ for _ in range(NREPS):
+ result = subprocess.run([bin, "-c", "pass"])
+ result.check_returncode()
+ t1 = time.time()
+ print(f"{(t1-t0)/NREPS:6.3f} {bin}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py
new file mode 100644
index 00000000000000..e0d18c89e75319
--- /dev/null
+++ b/Tools/scripts/umarshal.py
@@ -0,0 +1,328 @@
+# Implementat marshal.loads() in pure Python
+
+import ast
+
+from typing import Any
+
+
+class Type:
+ # Adapted from marshal.c
+ NULL = ord('0')
+ NONE = ord('N')
+ FALSE = ord('F')
+ TRUE = ord('T')
+ STOPITER = ord('S')
+ ELLIPSIS = ord('.')
+ INT = ord('i')
+ INT64 = ord('I')
+ FLOAT = ord('f')
+ BINARY_FLOAT = ord('g')
+ COMPLEX = ord('x')
+ BINARY_COMPLEX = ord('y')
+ LONG = ord('l')
+ STRING = ord('s')
+ INTERNED = ord('t')
+ REF = ord('r')
+ TUPLE = ord('(')
+ LIST = ord('[')
+ DICT = ord('{')
+ CODE = ord('c')
+ UNICODE = ord('u')
+ UNKNOWN = ord('?')
+ SET = ord('<')
+ FROZENSET = ord('>')
+ ASCII = ord('a')
+ ASCII_INTERNED = ord('A')
+ SMALL_TUPLE = ord(')')
+ SHORT_ASCII = ord('z')
+ SHORT_ASCII_INTERNED = ord('Z')
+
+
+FLAG_REF = 0x80 # with a type, add obj to index
+
+NULL = object() # marker
+
+# Cell kinds
+CO_FAST_LOCAL = 0x20
+CO_FAST_CELL = 0x40
+CO_FAST_FREE = 0x80
+
+
+class Code:
+ def __init__(self, **kwds: Any):
+ self.__dict__.update(kwds)
+
+ def __repr__(self) -> str:
+ return f"Code(**{self.__dict__})"
+
+ co_localsplusnames: tuple[str]
+ co_localspluskinds: tuple[int]
+
+ def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]:
+ varnames: list[str] = []
+ for name, kind in zip(self.co_localsplusnames,
+ self.co_localspluskinds):
+ if kind & select_kind:
+ varnames.append(name)
+ return tuple(varnames)
+
+ @property
+ def co_varnames(self) -> tuple[str, ...]:
+ return self.get_localsplus_names(CO_FAST_LOCAL)
+
+ @property
+ def co_cellvars(self) -> tuple[str, ...]:
+ return self.get_localsplus_names(CO_FAST_CELL)
+
+ @property
+ def co_freevars(self) -> tuple[str, ...]:
+ return self.get_localsplus_names(CO_FAST_FREE)
+
+ @property
+ def co_nlocals(self) -> int:
+ return len(self.co_varnames)
+
+
+class Reader:
+ # A fairly literal translation of the marshal reader.
+
+ def __init__(self, data: bytes):
+ self.data: bytes = data
+ self.end: int = len(self.data)
+ self.pos: int = 0
+ self.refs: list[Any] = []
+ self.level: int = 0
+
+ def r_string(self, n: int) -> bytes:
+ assert 0 <= n <= self.end - self.pos
+ buf = self.data[self.pos : self.pos + n]
+ self.pos += n
+ return buf
+
+ def r_byte(self) -> int:
+ buf = self.r_string(1)
+ return buf[0]
+
+ def r_short(self) -> int:
+ buf = self.r_string(2)
+ x = buf[0]
+ x |= buf[1] << 8
+ x |= -(x & (1<<15)) # Sign-extend
+ return x
+
+ def r_long(self) -> int:
+ buf = self.r_string(4)
+ x = buf[0]
+ x |= buf[1] << 8
+ x |= buf[2] << 16
+ x |= buf[3] << 24
+ x |= -(x & (1<<31)) # Sign-extend
+ return x
+
+ def r_long64(self) -> int:
+ buf = self.r_string(8)
+ x = buf[0]
+ x |= buf[1] << 8
+ x |= buf[2] << 16
+ x |= buf[3] << 24
+ x |= buf[1] << 32
+ x |= buf[1] << 40
+ x |= buf[1] << 48
+ x |= buf[1] << 56
+ x |= -(x & (1<<63)) # Sign-extend
+ return x
+
+ def r_PyLong(self) -> int:
+ n = self.r_long()
+ size = abs(n)
+ x = 0
+ # Pray this is right
+ for i in range(size):
+ x |= self.r_short() << i*15
+ if n < 0:
+ x = -x
+ return x
+
+ def r_float_bin(self) -> float:
+ buf = self.r_string(8)
+ import struct # Lazy import to avoid breaking UNIX build
+ return struct.unpack("d", buf)[0]
+
+ def r_float_str(self) -> float:
+ n = self.r_byte()
+ buf = self.r_string(n)
+ return ast.literal_eval(buf.decode("ascii"))
+
+ def r_ref_reserve(self, flag: int) -> int:
+ if flag:
+ idx = len(self.refs)
+ self.refs.append(None)
+ return idx
+ else:
+ return 0
+
+ def r_ref_insert(self, obj: Any, idx: int, flag: int) -> Any:
+ if flag:
+ self.refs[idx] = obj
+ return obj
+
+ def r_ref(self, obj: Any, flag: int) -> Any:
+ assert flag & FLAG_REF
+ self.refs.append(obj)
+ return obj
+
+ def r_object(self) -> Any:
+ old_level = self.level
+ try:
+ return self._r_object()
+ finally:
+ self.level = old_level
+
+ def _r_object(self) -> Any:
+ code = self.r_byte()
+ flag = code & FLAG_REF
+ type = code & ~FLAG_REF
+ # print(" "*self.level + f"{code} {flag} {type} {chr(type)!r}")
+ self.level += 1
+
+ def R_REF(obj: Any) -> Any:
+ if flag:
+ obj = self.r_ref(obj, flag)
+ return obj
+
+ match type:
+ case Type.NULL:
+ return NULL
+ case Type.NONE:
+ return None
+ case Type.ELLIPSIS:
+ return Ellipsis
+ case Type.FALSE:
+ return False
+ case Type.TRUE:
+ return True
+ case Type.INT:
+ return R_REF(self.r_long())
+ case Type.INT64:
+ return R_REF(self.r_long64())
+ case Type.LONG:
+ return R_REF(self.r_PyLong())
+ case Type.FLOAT:
+ return R_REF(self.r_float_str())
+ case Type.BINARY_FLOAT:
+ return R_REF(self.r_float_bin())
+ case Type.COMPLEX:
+ return R_REF(complex(self.r_float_str(),
+ self.r_float_str()))
+ case Type.BINARY_COMPLEX:
+ return R_REF(complex(self.r_float_bin(),
+ self.r_float_bin()))
+ case Type.STRING:
+ n = self.r_long()
+ return R_REF(self.r_string(n))
+ case Type.ASCII_INTERNED | Type.ASCII:
+ n = self.r_long()
+ return R_REF(self.r_string(n).decode("ascii"))
+ case Type.SHORT_ASCII_INTERNED | Type.SHORT_ASCII:
+ n = self.r_byte()
+ return R_REF(self.r_string(n).decode("ascii"))
+ case Type.INTERNED | Type.UNICODE:
+ n = self.r_long()
+ return R_REF(self.r_string(n).decode("utf8", "surrogatepass"))
+ case Type.SMALL_TUPLE:
+ n = self.r_byte()
+ idx = self.r_ref_reserve(flag)
+ retval: Any = tuple(self.r_object() for _ in range(n))
+ self.r_ref_insert(retval, idx, flag)
+ return retval
+ case Type.TUPLE:
+ n = self.r_long()
+ idx = self.r_ref_reserve(flag)
+ retval = tuple(self.r_object() for _ in range(n))
+ self.r_ref_insert(retval, idx, flag)
+ return retval
+ case Type.LIST:
+ n = self.r_long()
+ retval = R_REF([])
+ for _ in range(n):
+ retval.append(self.r_object())
+ return retval
+ case Type.DICT:
+ retval = R_REF({})
+ while True:
+ key = self.r_object()
+ if key == NULL:
+ break
+ val = self.r_object()
+ retval[key] = val
+ return retval
+ case Type.SET:
+ n = self.r_long()
+ retval = R_REF(set())
+ for _ in range(n):
+ v = self.r_object()
+ retval.add(v)
+ return retval
+ case Type.FROZENSET:
+ n = self.r_long()
+ s: set[Any] = set()
+ idx = self.r_ref_reserve(flag)
+ for _ in range(n):
+ v = self.r_object()
+ s.add(v)
+ retval = frozenset(s)
+ self.r_ref_insert(retval, idx, flag)
+ return retval
+ case Type.CODE:
+ retval = R_REF(Code())
+ retval.co_argcount = self.r_long()
+ retval.co_posonlyargcount = self.r_long()
+ retval.co_kwonlyargcount = self.r_long()
+ retval.co_stacksize = self.r_long()
+ retval.co_flags = self.r_long()
+ retval.co_code = self.r_object()
+ retval.co_consts = self.r_object()
+ retval.co_names = self.r_object()
+ retval.co_localsplusnames = self.r_object()
+ retval.co_localspluskinds = self.r_object()
+ retval.co_filename = self.r_object()
+ retval.co_name = self.r_object()
+ retval.co_qualname = self.r_object()
+ retval.co_firstlineno = self.r_long()
+ retval.co_linetable = self.r_object()
+ retval.co_endlinetable = self.r_object()
+ retval.co_columntable = self.r_object()
+ retval.co_exceptiontable = self.r_object()
+ return retval
+ case Type.REF:
+ n = self.r_long()
+ retval = self.refs[n]
+ assert retval is not None
+ return retval
+ case _:
+ breakpoint()
+ raise AssertionError(f"Unknown type {type} {chr(type)!r}")
+
+
+def loads(data: bytes) -> Any:
+ assert isinstance(data, bytes)
+ r = Reader(data)
+ return r.r_object()
+
+
+def main():
+ # Test
+ import marshal, pprint
+ sample = {'foo': {(42, "bar", 3.14)}}
+ data = marshal.dumps(sample)
+ retval = loads(data)
+ assert retval == sample, retval
+ sample = main.__code__
+ data = marshal.dumps(sample)
+ retval = loads(data)
+ assert isinstance(retval, Code), retval
+ pprint.pprint(retval.__dict__)
+
+
+if __name__ == "__main__":
+ main()