From 97c8fd6c46fead21fb78988c41cae53c23b27701 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 17 Nov 2021 14:56:14 -0800 Subject: [PATCH 01/17] Deep-freeze for Windows, step 1 --- PCbuild/pythoncore.vcxproj | 24 ++++++++++++++++++++++++ Python/frozen.c | 7 +------ Tools/scripts/freeze_modules.py | 2 +- deepfreeze.bat | 20 ++++++++++++++++++++ 4 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 deepfreeze.bat diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b65998186927b6..7f006aa39d60f4 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -500,6 +500,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Python/frozen.c b/Python/frozen.c index 1565c9a3d73f7b..bf9cd2e6f2c7f9 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -61,12 +61,7 @@ #include "frozen_modules/frozen_only.h" /* End includes */ -#ifdef MS_WINDOWS -/* Deepfreeze isn't supported on Windows yet. */ -#define GET_CODE(name) NULL -#else #define GET_CODE(name) _Py_get_##name##_toplevel -#endif /* Start extern declarations */ extern PyObject *_Py_get_importlib__bootstrap_toplevel(void); @@ -108,7 +103,7 @@ static const struct _frozen bootstrap_modules[] = { static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), GET_CODE(abc)}, - {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, + // {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, {"io", _Py_M__io, (int)sizeof(_Py_M__io), GET_CODE(io)}, /* stdlib - startup, with site */ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index ccea4e11ab6ca3..18fa3d0b986b32 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -75,7 +75,7 @@ def find_tool(): ]), ('stdlib - startup, without site (python -S)', [ 'abc', - 'codecs', + # 'codecs', # TODO: Windows name conflict with Python\codecs.c # For now we do not freeze the encodings, due # to the noise all # those extra modules add to the text printed during the build. # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) diff --git a/deepfreeze.bat b/deepfreeze.bat new file mode 100644 index 00000000000000..3ae3dec44c3703 --- /dev/null +++ b/deepfreeze.bat @@ -0,0 +1,20 @@ + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/importlib\_bootstrap.py -m importlib._bootstrap -o Python\deepfreeze\importlib._bootstrap.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/importlib\_bootstrap_external.py -m importlib._bootstrap_external -o Python\deepfreeze\importlib._bootstrap_external.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/zipimport.py -m zipimport -o Python\deepfreeze\zipimport.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/abc.py -m abc -o Python\deepfreeze\abc.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/codecs.py -m codecs -o Python\deepfreeze\codecs.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/io.py -m io -o Python\deepfreeze\io.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/_collections_abc.py -m _collections_abc -o Python\deepfreeze\_collections_abc.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/_sitebuiltins.py -m _sitebuiltins -o Python\deepfreeze\_sitebuiltins.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/genericpath.py -m genericpath -o Python\deepfreeze\genericpath.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/ntpath.py -m ntpath -o Python\deepfreeze\ntpath.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/posixpath.py -m posixpath -o Python\deepfreeze\posixpath.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/os.py -m os -o Python\deepfreeze\os.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/site.py -m site -o Python\deepfreeze\site.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/stat.py -m stat -o Python\deepfreeze\stat.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__hello__.py -m __hello__ -o Python\deepfreeze\__hello__.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\__init__.py -m __phello__ -o Python\deepfreeze\__phello__.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\ham\__init__.py -m __phello__.ham -o Python\deepfreeze\__phello__.ham.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\ham\eggs.py -m __phello__.ham.eggs -o Python\deepfreeze\__phello__.ham.eggs.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\spam.py -m __phello__.spam -o Python\deepfreeze\__phello__.spam.c + pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Tools/freeze/flag.py -m frozen_only -o Python\deepfreeze\frozen_only.c \ No newline at end of file From 65ee8fefc8fca3704e991c65278bcbd5fed70705 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 17 Nov 2021 16:58:49 -0800 Subject: [PATCH 02/17] Support reading marshalled data from a frozen .h file --- Tools/scripts/deepfreeze.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 074127f9492988..0cab5e19fa009a 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -1,8 +1,11 @@ import argparse +import ast import builtins import collections import contextlib +import marshal import os +import re import sys import time import types @@ -367,8 +370,33 @@ def generate(self, name: str, obj: object) -> str: } """ +FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */" + +FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" + + +def is_frozen_header(source: str) -> bool: + # TODO: What if _freeze_module.c has a different filename? + return source.startswith(FROZEN_COMMENT) + + +def decode_frozen_data(source: str) -> types.CodeType: + lines = source.splitlines() + while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None: + del lines[0] + while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: + del lines[-1] + values: tuple[int, ...] = ast.literal_eval("".join(lines)) + data = bytes(values) + # TODO: Write a marshal.unloads() implementation in pure Python + return marshal.loads(data) + + def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None: - code = compile(source, filename, "exec") + if is_frozen_header(source): + code = decode_frozen_data(source) + else: + code = compile(source, filename, "exec") printer = Printer(file) printer.generate("toplevel", code) printer.write("") From de230cb3020c4d47e421323d05178bd996f31f94 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 17 Nov 2021 19:53:42 -0800 Subject: [PATCH 03/17] Add umarshal.py (3.11 marshal.loads() in pure Python) --- Tools/scripts/deepfreeze.py | 11 +- Tools/scripts/umarshal.py | 328 ++++++++++++++++++++++++++++++++++++ 2 files changed, 334 insertions(+), 5 deletions(-) create mode 100644 Tools/scripts/umarshal.py diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 0cab5e19fa009a..6844f0d6904222 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -11,6 +11,8 @@ import types import typing +import umarshal + verbose = False @@ -58,7 +60,8 @@ def get_localsplus_counts(code: types.CodeType, nplaincellvars += 1 elif kind & CO_FAST_FREE: nfreevars += 1 - assert nlocals == len(code.co_varnames) == code.co_nlocals + assert nlocals == len(code.co_varnames) == code.co_nlocals, \ + (nlocals, len(code.co_varnames), code.co_nlocals) assert ncellvars == len(code.co_cellvars) assert nfreevars == len(code.co_freevars) assert len(names) == nlocals + nplaincellvars + nfreevars @@ -329,7 +332,7 @@ def generate(self, name: str, obj: object) -> str: return self.cache[key] self.misses += 1 match obj: - case types.CodeType() as code: + case types.CodeType() | umarshal.Code() as code: val = self.generate_code(name, code) case tuple(t): val = self.generate_tuple(name, t) @@ -376,7 +379,6 @@ def generate(self, name: str, obj: object) -> str: def is_frozen_header(source: str) -> bool: - # TODO: What if _freeze_module.c has a different filename? return source.startswith(FROZEN_COMMENT) @@ -388,8 +390,7 @@ def decode_frozen_data(source: str) -> types.CodeType: del lines[-1] values: tuple[int, ...] = ast.literal_eval("".join(lines)) data = bytes(values) - # TODO: Write a marshal.unloads() implementation in pure Python - return marshal.loads(data) + return umarshal.loads(data) def generate(source: str, filename: str, modname: str, file: typing.TextIO) -> None: diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py new file mode 100644 index 00000000000000..3fda65cdf201ce --- /dev/null +++ b/Tools/scripts/umarshal.py @@ -0,0 +1,328 @@ +# Implementat marshal.loads() in pure Python + +import ast +import struct + +from typing import Any + + +class Type: + # Adapted from marshal.c + NULL = ord('0') + NONE = ord('N') + FALSE = ord('F') + TRUE = ord('T') + STOPITER = ord('S') + ELLIPSIS = ord('.') + INT = ord('i') + INT64 = ord('I') + FLOAT = ord('f') + BINARY_FLOAT = ord('g') + COMPLEX = ord('x') + BINARY_COMPLEX = ord('y') + LONG = ord('l') + STRING = ord('s') + INTERNED = ord('t') + REF = ord('r') + TUPLE = ord('(') + LIST = ord('[') + DICT = ord('{') + CODE = ord('c') + UNICODE = ord('u') + UNKNOWN = ord('?') + SET = ord('<') + FROZENSET = ord('>') + ASCII = ord('a') + ASCII_INTERNED = ord('A') + SMALL_TUPLE = ord(')') + SHORT_ASCII = ord('z') + SHORT_ASCII_INTERNED = ord('Z') + + +FLAG_REF = 0x80 # with a type, add obj to index + +NULL = object() # marker + +# Cell kinds +CO_FAST_LOCAL = 0x20 +CO_FAST_CELL = 0x40 +CO_FAST_FREE = 0x80 + + +class Code: + def __init__(self, **kwds: Any): + self.__dict__.update(kwds) + + def __repr__(self) -> str: + return f"Code(**{self.__dict__})" + + co_localsplusnames: tuple[str] + co_localspluskinds: tuple[int] + + def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: + varnames: list[str] = [] + for name, kind in zip(self.co_localsplusnames, + self.co_localspluskinds): + if kind & select_kind: + varnames.append(name) + return tuple(varnames) + + @property + def co_varnames(self) -> tuple[str, ...]: + return self.get_localsplus_names(CO_FAST_LOCAL) + + @property + def co_cellvars(self) -> tuple[str, ...]: + return self.get_localsplus_names(CO_FAST_CELL) + + @property + def co_freevars(self) -> tuple[str, ...]: + return self.get_localsplus_names(CO_FAST_FREE) + + @property + def co_nlocals(self) -> int: + return len(self.co_varnames) + + +class Reader: + # A fairly literal translation of the marshal reader. + + def __init__(self, data: bytes): + self.data: bytes = data + self.end: int = len(self.data) + self.pos: int = 0 + self.refs: list[Any] = [] + self.level: int = 0 + + def r_string(self, n: int) -> bytes: + assert 0 <= n <= self.end - self.pos + buf = self.data[self.pos : self.pos + n] + self.pos += n + return buf + + def r_byte(self) -> int: + buf = self.r_string(1) + return buf[0] + + def r_short(self) -> int: + buf = self.r_string(2) + x = buf[0] + x |= buf[1] << 8 + x |= -(x & (1<<15)) # Sign-extend + return x + + def r_long(self) -> int: + buf = self.r_string(4) + x = buf[0] + x |= buf[1] << 8 + x |= buf[2] << 16 + x |= buf[3] << 24 + x |= -(x & (1<<31)) # Sign-extend + return x + + def r_long64(self) -> int: + buf = self.r_string(8) + x = buf[0] + x |= buf[1] << 8 + x |= buf[2] << 16 + x |= buf[3] << 24 + x |= buf[1] << 32 + x |= buf[1] << 40 + x |= buf[1] << 48 + x |= buf[1] << 56 + x |= -(x & (1<<63)) # Sign-extend + return x + + def r_PyLong(self) -> int: + n = self.r_long() + size = abs(n) + x = 0 + # Pray this is right + for i in range(size): + x |= self.r_short() << i*15 + if n < 0: + x = -x + return x + + def r_float_bin(self) -> float: + buf = self.r_string(8) + return struct.unpack("d", buf)[0] + + def r_float_str(self) -> float: + n = self.r_byte() + buf = self.r_string(n) + return ast.literal_eval(buf.decode("ascii")) + + def r_ref_reserve(self, flag: int) -> int: + if flag: + idx = len(self.refs) + self.refs.append(None) + return idx + else: + return 0 + + def r_ref_insert(self, obj: Any, idx: int, flag: int) -> Any: + if flag: + self.refs[idx] = obj + return obj + + def r_ref(self, obj: Any, flag: int) -> Any: + assert flag & FLAG_REF + self.refs.append(obj) + return obj + + def r_object(self) -> Any: + old_level = self.level + try: + return self._r_object() + finally: + self.level = old_level + + def _r_object(self) -> Any: + code = self.r_byte() + flag = code & FLAG_REF + type = code & ~FLAG_REF + # print(" "*self.level + f"{code} {flag} {type} {chr(type)!r}") + self.level += 1 + + def R_REF(obj: Any) -> Any: + if flag: + obj = self.r_ref(obj, flag) + return obj + + match type: + case Type.NULL: + return NULL + case Type.NONE: + return None + case Type.ELLIPSIS: + return Ellipsis + case Type.FALSE: + return False + case Type.TRUE: + return True + case Type.INT: + return R_REF(self.r_long()) + case Type.INT64: + return R_REF(self.r_long64()) + case Type.LONG: + return R_REF(self.r_PyLong()) + case Type.FLOAT: + return R_REF(self.r_float_str()) + case Type.BINARY_FLOAT: + return R_REF(self.r_float_bin()) + case Type.COMPLEX: + return R_REF(complex(self.r_float_str(), + self.r_float_str())) + case Type.BINARY_COMPLEX: + return R_REF(complex(self.r_float_bin(), + self.r_float_bin())) + case Type.STRING: + n = self.r_long() + return R_REF(self.r_string(n)) + case Type.ASCII_INTERNED | Type.ASCII: + n = self.r_long() + return R_REF(self.r_string(n).decode("ascii")) + case Type.SHORT_ASCII_INTERNED | Type.SHORT_ASCII: + n = self.r_byte() + return R_REF(self.r_string(n).decode("ascii")) + case Type.INTERNED | Type.UNICODE: + n = self.r_long() + return R_REF(self.r_string(n).decode("utf8", "surrogatepass")) + case Type.SMALL_TUPLE: + n = self.r_byte() + idx = self.r_ref_reserve(flag) + retval: Any = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + case Type.TUPLE: + n = self.r_long() + idx = self.r_ref_reserve(flag) + retval = tuple(self.r_object() for _ in range(n)) + self.r_ref_insert(retval, idx, flag) + return retval + case Type.LIST: + n = self.r_long() + retval = R_REF([]) + for _ in range(n): + retval.append(self.r_object()) + return retval + case Type.DICT: + retval = R_REF({}) + while True: + key = self.r_object() + if key == NULL: + break + val = self.r_object() + retval[key] = val + return retval + case Type.SET: + n = self.r_long() + retval = R_REF(set()) + for _ in range(n): + v = self.r_object() + retval.add(v) + return retval + case Type.FROZENSET: + n = self.r_long() + s: set[Any] = set() + idx = self.r_ref_reserve(flag) + for _ in range(n): + v = self.r_object() + s.add(v) + retval = frozenset(s) + self.r_ref_insert(retval, idx, flag) + return retval + case Type.CODE: + retval = R_REF(Code()) + retval.co_argcount = self.r_long() + retval.co_posonlyargcount = self.r_long() + retval.co_kwonlyargcount = self.r_long() + retval.co_stacksize = self.r_long() + retval.co_flags = self.r_long() + retval.co_code = self.r_object() + retval.co_consts = self.r_object() + retval.co_names = self.r_object() + retval.co_localsplusnames = self.r_object() + retval.co_localspluskinds = self.r_object() + retval.co_filename = self.r_object() + retval.co_name = self.r_object() + retval.co_qualname = self.r_object() + retval.co_firstlineno = self.r_long() + retval.co_linetable = self.r_object() + retval.co_endlinetable = self.r_object() + retval.co_columntable = self.r_object() + retval.co_exceptiontable = self.r_object() + return retval + case Type.REF: + n = self.r_long() + retval = self.refs[n] + assert retval is not None + return retval + case _: + breakpoint() + raise AssertionError(f"Unknown type {type} {chr(type)!r}") + + +def loads(data: bytes) -> Any: + assert isinstance(data, bytes) + r = Reader(data) + return r.r_object() + + +def main(): + # Test + import marshal, pprint + sample = {'foo': {(42, "bar", 3.14)}} + data = marshal.dumps(sample) + retval = loads(data) + assert retval == sample, retval + sample = main.__code__ + data = marshal.dumps(sample) + retval = loads(data) + assert isinstance(retval, Code), retval + pprint.pprint(retval.__dict__) + + +if __name__ == "__main__": + main() From 87c6cb4f9ead95d35ca61d32567f4c611d97bd48 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 18 Nov 2021 15:23:48 -0800 Subject: [PATCH 04/17] Generate .c files in _freeze_module.vcxproj (But needs work, to avoid recompiling each time.) --- PCbuild/_freeze_module.vcxproj | 2 ++ PCbuild/find_python.bat | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index d33e07c54b8c9c..109b0b6349ba4c 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -337,6 +337,8 @@ + + = (3, 8)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found @rem If py.exe finds a recent enough version, use that one -@for %%p in (3.9 3.8) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% From 8b46af7b4197c79616fd70f736439bfe608c084f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 12:22:32 -0800 Subject: [PATCH 05/17] Avoid copying unchanged deep-frozen files The recipe is copied from how unchanged frozen files are treated. --- PCbuild/_freeze_module.vcxproj | 38 +++++++++++++++++++++----- PCbuild/_freeze_module.vcxproj.filters | 3 -- Tools/scripts/freeze_modules.py | 2 ++ 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 109b0b6349ba4c..2702bfa1d4437a 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -234,101 +234,115 @@ importlib._bootstrap $(IntDir)importlib._bootstrap.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h + $(PySourcePath)Python/deepfreeze/importlib._bootstrap.c importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h + $(PySourcePath)Python/deepfreeze/importlib._bootstrap_external.c zipimport $(IntDir)zipimport.g.h $(PySourcePath)Python\frozen_modules\zipimport.h + $(PySourcePath)Python/deepfreeze/zipimport.c abc $(IntDir)abc.g.h $(PySourcePath)Python\frozen_modules\abc.h - - - codecs - $(IntDir)codecs.g.h - $(PySourcePath)Python\frozen_modules\codecs.h + $(PySourcePath)Python/deepfreeze/abc.c io $(IntDir)io.g.h $(PySourcePath)Python\frozen_modules\io.h + $(PySourcePath)Python/deepfreeze/io.c _collections_abc $(IntDir)_collections_abc.g.h $(PySourcePath)Python\frozen_modules\_collections_abc.h + $(PySourcePath)Python/deepfreeze/_collections_abc.c _sitebuiltins $(IntDir)_sitebuiltins.g.h $(PySourcePath)Python\frozen_modules\_sitebuiltins.h + $(PySourcePath)Python/deepfreeze/_sitebuiltins.c genericpath $(IntDir)genericpath.g.h $(PySourcePath)Python\frozen_modules\genericpath.h + $(PySourcePath)Python/deepfreeze/genericpath.c ntpath $(IntDir)ntpath.g.h $(PySourcePath)Python\frozen_modules\ntpath.h + $(PySourcePath)Python/deepfreeze/ntpath.c posixpath $(IntDir)posixpath.g.h $(PySourcePath)Python\frozen_modules\posixpath.h + $(PySourcePath)Python/deepfreeze/posixpath.c os $(IntDir)os.g.h $(PySourcePath)Python\frozen_modules\os.h + $(PySourcePath)Python/deepfreeze/os.c site $(IntDir)site.g.h $(PySourcePath)Python\frozen_modules\site.h + $(PySourcePath)Python/deepfreeze/site.c stat $(IntDir)stat.g.h $(PySourcePath)Python\frozen_modules\stat.h + $(PySourcePath)Python/deepfreeze/stat.c __hello__ $(IntDir)__hello__.g.h $(PySourcePath)Python\frozen_modules\__hello__.h + $(PySourcePath)Python/deepfreeze/__hello__.c __phello__ $(IntDir)__phello__.g.h $(PySourcePath)Python\frozen_modules\__phello__.h + $(PySourcePath)Python/deepfreeze/__phello__.c __phello__.ham $(IntDir)__phello__.ham.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.h + $(PySourcePath)Python/deepfreeze/__phello__.ham.c __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h + $(PySourcePath)Python/deepfreeze/__phello__.ham.eggs.c __phello__.spam $(IntDir)__phello__.spam.g.h $(PySourcePath)Python\frozen_modules\__phello__.spam.h + $(PySourcePath)Python/deepfreeze/__phello__.spam.c frozen_only $(IntDir)frozen_only.g.h $(PySourcePath)Python\frozen_modules\frozen_only.h + $(PySourcePath)Python/deepfreeze/frozen_only.c @@ -337,8 +351,6 @@ - - + + + + + + + + + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 5894909e0fbe1e..c9d58a25b79475 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -28,9 +28,6 @@ Python Files - - Python Files - Python Files diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 18fa3d0b986b32..375582fffb4ed4 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -720,11 +720,13 @@ def regen_pcbuild(modules): for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) + deepoutfile = f"Python/deepfreeze/{src.id}.c" intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') projlines.append(f' $(PySourcePath){header}') + projlines.append(f' $(PySourcePath){deepoutfile}') projlines.append(f' ') filterlines.append(f' ') From 1bc777f607ccbea8934c2a6df402db80722f420c Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 15:05:30 -0800 Subject: [PATCH 06/17] Properly generate lines in pythoncore.vcxproj; add codecs back --- PCbuild/_freeze_module.vcxproj | 44 +++++++++++++++----------- PCbuild/_freeze_module.vcxproj.filters | 3 ++ PCbuild/pythoncore.vcxproj | 40 +++++++++++------------ Python/frozen.c | 8 ++--- Tools/scripts/freeze_modules.py | 23 ++++++++++++-- 5 files changed, 72 insertions(+), 46 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 2702bfa1d4437a..b57ca6bc75e135 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -234,115 +234,121 @@ importlib._bootstrap $(IntDir)importlib._bootstrap.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h - $(PySourcePath)Python/deepfreeze/importlib._bootstrap.c + $(PySourcePath)Python\deepfreeze\importlib._bootstrap.c importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h - $(PySourcePath)Python/deepfreeze/importlib._bootstrap_external.c + $(PySourcePath)Python\deepfreeze\importlib._bootstrap_external.c zipimport $(IntDir)zipimport.g.h $(PySourcePath)Python\frozen_modules\zipimport.h - $(PySourcePath)Python/deepfreeze/zipimport.c + $(PySourcePath)Python\deepfreeze\zipimport.c abc $(IntDir)abc.g.h $(PySourcePath)Python\frozen_modules\abc.h - $(PySourcePath)Python/deepfreeze/abc.c + $(PySourcePath)Python\deepfreeze\abc.c + + + codecs + $(IntDir)codecs.g.h + $(PySourcePath)Python\frozen_modules\codecs.h + $(PySourcePath)Python\deepfreeze\codecs_.c io $(IntDir)io.g.h $(PySourcePath)Python\frozen_modules\io.h - $(PySourcePath)Python/deepfreeze/io.c + $(PySourcePath)Python\deepfreeze\io.c _collections_abc $(IntDir)_collections_abc.g.h $(PySourcePath)Python\frozen_modules\_collections_abc.h - $(PySourcePath)Python/deepfreeze/_collections_abc.c + $(PySourcePath)Python\deepfreeze\_collections_abc.c _sitebuiltins $(IntDir)_sitebuiltins.g.h $(PySourcePath)Python\frozen_modules\_sitebuiltins.h - $(PySourcePath)Python/deepfreeze/_sitebuiltins.c + $(PySourcePath)Python\deepfreeze\_sitebuiltins.c genericpath $(IntDir)genericpath.g.h $(PySourcePath)Python\frozen_modules\genericpath.h - $(PySourcePath)Python/deepfreeze/genericpath.c + $(PySourcePath)Python\deepfreeze\genericpath.c ntpath $(IntDir)ntpath.g.h $(PySourcePath)Python\frozen_modules\ntpath.h - $(PySourcePath)Python/deepfreeze/ntpath.c + $(PySourcePath)Python\deepfreeze\ntpath.c posixpath $(IntDir)posixpath.g.h $(PySourcePath)Python\frozen_modules\posixpath.h - $(PySourcePath)Python/deepfreeze/posixpath.c + $(PySourcePath)Python\deepfreeze\posixpath.c os $(IntDir)os.g.h $(PySourcePath)Python\frozen_modules\os.h - $(PySourcePath)Python/deepfreeze/os.c + $(PySourcePath)Python\deepfreeze\os.c site $(IntDir)site.g.h $(PySourcePath)Python\frozen_modules\site.h - $(PySourcePath)Python/deepfreeze/site.c + $(PySourcePath)Python\deepfreeze\site.c stat $(IntDir)stat.g.h $(PySourcePath)Python\frozen_modules\stat.h - $(PySourcePath)Python/deepfreeze/stat.c + $(PySourcePath)Python\deepfreeze\stat.c __hello__ $(IntDir)__hello__.g.h $(PySourcePath)Python\frozen_modules\__hello__.h - $(PySourcePath)Python/deepfreeze/__hello__.c + $(PySourcePath)Python\deepfreeze\__hello__.c __phello__ $(IntDir)__phello__.g.h $(PySourcePath)Python\frozen_modules\__phello__.h - $(PySourcePath)Python/deepfreeze/__phello__.c + $(PySourcePath)Python\deepfreeze\__phello__.c __phello__.ham $(IntDir)__phello__.ham.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.h - $(PySourcePath)Python/deepfreeze/__phello__.ham.c + $(PySourcePath)Python\deepfreeze\__phello__.ham.c __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h - $(PySourcePath)Python/deepfreeze/__phello__.ham.eggs.c + $(PySourcePath)Python\deepfreeze\__phello__.ham.eggs.c __phello__.spam $(IntDir)__phello__.spam.g.h $(PySourcePath)Python\frozen_modules\__phello__.spam.h - $(PySourcePath)Python/deepfreeze/__phello__.spam.c + $(PySourcePath)Python\deepfreeze\__phello__.spam.c frozen_only $(IntDir)frozen_only.g.h $(PySourcePath)Python\frozen_modules\frozen_only.h - $(PySourcePath)Python/deepfreeze/frozen_only.c + $(PySourcePath)Python\deepfreeze\frozen_only.c diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index c9d58a25b79475..5894909e0fbe1e 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -28,6 +28,9 @@ Python Files + + Python Files + Python Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7f006aa39d60f4..5cbcc35883ca4b 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -502,26 +502,26 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + diff --git a/Python/frozen.c b/Python/frozen.c index bf9cd2e6f2c7f9..1441a3cb421913 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -75,7 +75,7 @@ extern PyObject *_Py_get__sitebuiltins_toplevel(void); extern PyObject *_Py_get_genericpath_toplevel(void); extern PyObject *_Py_get_ntpath_toplevel(void); extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_posixpath_toplevel(void); +extern PyObject *_Py_get_ntpath_toplevel(void); extern PyObject *_Py_get_os_toplevel(void); extern PyObject *_Py_get_site_toplevel(void); extern PyObject *_Py_get_stat_toplevel(void); @@ -103,7 +103,7 @@ static const struct _frozen bootstrap_modules[] = { static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), GET_CODE(abc)}, - // {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, + {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, {"io", _Py_M__io, (int)sizeof(_Py_M__io), GET_CODE(io)}, /* stdlib - startup, with site */ @@ -112,7 +112,7 @@ static const struct _frozen stdlib_modules[] = { {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), GET_CODE(genericpath)}, {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)}, {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, - {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, + {"os.path", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)}, {"os", _Py_M__os, (int)sizeof(_Py_M__os), GET_CODE(os)}, {"site", _Py_M__site, (int)sizeof(_Py_M__site), GET_CODE(site)}, {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), GET_CODE(stat)}, @@ -139,7 +139,7 @@ const struct _frozen *_PyImport_FrozenTest = test_modules; static const struct _module_alias aliases[] = { {"_frozen_importlib", "importlib._bootstrap"}, {"_frozen_importlib_external", "importlib._bootstrap_external"}, - {"os.path", "posixpath"}, + {"os.path", "ntpath"}, {"__hello_alias__", "__hello__"}, {"__phello_alias__", "__hello__"}, {"__phello_alias__.spam", "__hello__"}, diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 375582fffb4ed4..be81c14a21a05c 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -11,7 +11,6 @@ import platform import subprocess import sys -import textwrap import time from update_file import updating_file_with_tmpfile, update_file_with_tmpfile @@ -55,6 +54,7 @@ def find_tool(): MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj') PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters') +PCBUILD_PYTHONCORE = os.path.join(ROOT_DIR, 'PCbuild', 'pythoncore.vcxproj') OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath' @@ -75,7 +75,7 @@ def find_tool(): ]), ('stdlib - startup, without site (python -S)', [ 'abc', - # 'codecs', # TODO: Windows name conflict with Python\codecs.c + 'codecs', # For now we do not freeze the encodings, due # to the noise all # those extra modules add to the text printed during the build. # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) @@ -717,10 +717,14 @@ def regen_makefile(modules): def regen_pcbuild(modules): projlines = [] filterlines = [] + corelines = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) - deepoutfile = f"Python/deepfreeze/{src.id}.c" + deepbase = src.id + if deepbase == "codecs": + deepbase = "codecs_" # Annoying exception (duplicate file) + deepoutfile = f"Python\\deepfreeze\\{deepbase}.c" intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' projlines.append(f' ') projlines.append(f' {src.frozenid}') @@ -733,6 +737,8 @@ def regen_pcbuild(modules): filterlines.append(' Python Files') filterlines.append(' ') + corelines.append(f' ') + print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): lines = infile.readlines() @@ -755,6 +761,17 @@ def regen_pcbuild(modules): PCBUILD_FILTERS, ) outfile.writelines(lines) + print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}') + with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + corelines, + PCBUILD_FILTERS, + ) + outfile.writelines(lines) ####################################### From 97187ff107c2dcbe2a1677f18ce34408b2bae0cf Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 15:36:49 -0800 Subject: [PATCH 07/17] Try not to break UNIX builds by lazy-importing struct --- Tools/scripts/umarshal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/umarshal.py b/Tools/scripts/umarshal.py index 3fda65cdf201ce..e0d18c89e75319 100644 --- a/Tools/scripts/umarshal.py +++ b/Tools/scripts/umarshal.py @@ -1,7 +1,6 @@ # Implementat marshal.loads() in pure Python import ast -import struct from typing import Any @@ -146,6 +145,7 @@ def r_PyLong(self) -> int: def r_float_bin(self) -> float: buf = self.r_string(8) + import struct # Lazy import to avoid breaking UNIX build return struct.unpack("d", buf)[0] def r_float_str(self) -> float: From 8eeaea066ecb042a254cf4caef922fb2da631966 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 15:51:41 -0800 Subject: [PATCH 08/17] Remove temporary .bat file --- deepfreeze.bat | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 deepfreeze.bat diff --git a/deepfreeze.bat b/deepfreeze.bat deleted file mode 100644 index 3ae3dec44c3703..00000000000000 --- a/deepfreeze.bat +++ /dev/null @@ -1,20 +0,0 @@ - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/importlib\_bootstrap.py -m importlib._bootstrap -o Python\deepfreeze\importlib._bootstrap.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/importlib\_bootstrap_external.py -m importlib._bootstrap_external -o Python\deepfreeze\importlib._bootstrap_external.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/zipimport.py -m zipimport -o Python\deepfreeze\zipimport.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/abc.py -m abc -o Python\deepfreeze\abc.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/codecs.py -m codecs -o Python\deepfreeze\codecs.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/io.py -m io -o Python\deepfreeze\io.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/_collections_abc.py -m _collections_abc -o Python\deepfreeze\_collections_abc.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/_sitebuiltins.py -m _sitebuiltins -o Python\deepfreeze\_sitebuiltins.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/genericpath.py -m genericpath -o Python\deepfreeze\genericpath.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/ntpath.py -m ntpath -o Python\deepfreeze\ntpath.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/posixpath.py -m posixpath -o Python\deepfreeze\posixpath.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/os.py -m os -o Python\deepfreeze\os.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/site.py -m site -o Python\deepfreeze\site.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/stat.py -m stat -o Python\deepfreeze\stat.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__hello__.py -m __hello__ -o Python\deepfreeze\__hello__.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\__init__.py -m __phello__ -o Python\deepfreeze\__phello__.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\ham\__init__.py -m __phello__.ham -o Python\deepfreeze\__phello__.ham.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\ham\eggs.py -m __phello__.ham.eggs -o Python\deepfreeze\__phello__.ham.eggs.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Lib/__phello__\spam.py -m __phello__.spam -o Python\deepfreeze\__phello__.spam.c - pcbuild\amd64\python.exe Tools\scripts\deepfreeze.py Tools/freeze/flag.py -m frozen_only -o Python\deepfreeze\frozen_only.c \ No newline at end of file From d332f16998b67c9055068880ebdf0be5f8f10cb2 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 15:56:32 -0800 Subject: [PATCH 09/17] Change ntpath back to posixpath in frozen.c --- Python/frozen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/frozen.c b/Python/frozen.c index 1441a3cb421913..147ecd2c96949e 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -112,7 +112,7 @@ static const struct _frozen stdlib_modules[] = { {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), GET_CODE(genericpath)}, {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)}, {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, - {"os.path", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)}, + {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, {"os", _Py_M__os, (int)sizeof(_Py_M__os), GET_CODE(os)}, {"site", _Py_M__site, (int)sizeof(_Py_M__site), GET_CODE(site)}, {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), GET_CODE(stat)}, @@ -139,7 +139,7 @@ const struct _frozen *_PyImport_FrozenTest = test_modules; static const struct _module_alias aliases[] = { {"_frozen_importlib", "importlib._bootstrap"}, {"_frozen_importlib_external", "importlib._bootstrap_external"}, - {"os.path", "ntpath"}, + {"os.path", "posixpath"}, {"__hello_alias__", "__hello__"}, {"__phello_alias__", "__hello__"}, {"__phello_alias__.spam", "__hello__"}, From 3e7c3acda2dfe7d72c87ccba9e10ec6cc34dfde5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 15:59:48 -0800 Subject: [PATCH 10/17] One more nt->posix --- Python/frozen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/frozen.c b/Python/frozen.c index 147ecd2c96949e..9f43db70886f79 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -75,7 +75,7 @@ extern PyObject *_Py_get__sitebuiltins_toplevel(void); extern PyObject *_Py_get_genericpath_toplevel(void); extern PyObject *_Py_get_ntpath_toplevel(void); extern PyObject *_Py_get_posixpath_toplevel(void); -extern PyObject *_Py_get_ntpath_toplevel(void); +extern PyObject *_Py_get_posixpath_toplevel(void); extern PyObject *_Py_get_os_toplevel(void); extern PyObject *_Py_get_site_toplevel(void); extern PyObject *_Py_get_stat_toplevel(void); From 871f28bc5f6175744c15cee0fc387af6f1553b84 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 20 Nov 2021 00:07:02 +0000 Subject: [PATCH 11/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst diff --git a/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst new file mode 100644 index 00000000000000..0bbb577a539159 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst @@ -0,0 +1 @@ +Implement changes to build with deep-frozen modules on Windows. \ No newline at end of file From b52ad634831f70babe951a1ef4ee562cc382f033 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 19 Nov 2021 17:18:09 -0800 Subject: [PATCH 12/17] Fix by @zooba to get better error when no Python 3.10 is found Co-authored-by: Steve Dower --- PCbuild/find_python.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index 35b5601ff3e4ca..a9f14c5277ffa0 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -31,7 +31,7 @@ @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) @rem If we have Python in externals, use that one -@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" ("%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul) && (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") && (set _Py_Python_Source=found in externals directory) && goto :found || rmdir /Q /S "%_Py_EXTERNALS_DIR%\pythonx86" @rem If HOST_PYTHON is recent enough, use that @if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -Ec "import sys; assert sys.version_info[:2] >= (3, 10)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found From aea38e04cfd2929288300acb1702cad462792f90 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 20 Nov 2021 08:02:32 -0800 Subject: [PATCH 13/17] Generate integers for 15- and 30-bit digits This fixes the x86 build --- Tools/scripts/deepfreeze.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 6844f0d6904222..7de8cb0d287bf3 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -280,14 +280,7 @@ def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: self.write(item + ",") return f"& {name}._object.ob_base.ob_base" - def generate_int(self, name: str, i: int) -> str: - maxint = sys.maxsize - if maxint == 2**31 - 1: - digit = 2**15 - elif maxint == 2**63 - 1: - digit = 2**30 - else: - assert False, f"What int size is this system?!? {maxint=}" + def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: sign = -1 if i < 0 else 0 if i == 0 else +1 i = abs(i) digits: list[int] = [] @@ -304,6 +297,20 @@ def generate_int(self, name: str, i: int) -> str: if digits: ds = ", ".join(map(str, digits)) self.write(f".ob_digit = {{ {ds} }},") + + def generate_int(self, name: str, i: int) -> str: + if abs(i) < 2**15: + self._generate_int_for_bits(name, i, 2**15) + else: + connective = "if" + for bits_in_digit in 15, 30: + self.write(f"#{connective} PYLONG_BITS_IN_DIGIT == {bits_in_digit}") + self._generate_int_for_bits(name, i, 2**bits_in_digit) + connective = "elif" + self.write("#else") + self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"') + self.write("#endif") + # If neither clause applies, it won't compile return f"& {name}.ob_base.ob_base" def generate_float(self, name: str, x: float) -> str: From a7415ed78576572082c2f7f6ae144c0e4388ca39 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 20 Nov 2021 11:40:14 -0800 Subject: [PATCH 14/17] Give generated .c files a "df." prefix Also use a separate intermediate file for deepfreeze. --- PCbuild/_freeze_module.vcxproj | 66 +++++++++++++++++++++------------ PCbuild/pythoncore.vcxproj | 40 ++++++++++---------- Tools/scripts/freeze_modules.py | 6 +-- 3 files changed, 66 insertions(+), 46 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index b57ca6bc75e135..288302f8f93579 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -234,121 +234,141 @@ importlib._bootstrap $(IntDir)importlib._bootstrap.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h - $(PySourcePath)Python\deepfreeze\importlib._bootstrap.c + $(IntDir)importlib._bootstrap.g.c + $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap.c importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h - $(PySourcePath)Python\deepfreeze\importlib._bootstrap_external.c + $(IntDir)importlib._bootstrap_external.g.c + $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap_external.c zipimport $(IntDir)zipimport.g.h $(PySourcePath)Python\frozen_modules\zipimport.h - $(PySourcePath)Python\deepfreeze\zipimport.c + $(IntDir)zipimport.g.c + $(PySourcePath)Python\deepfreeze\df.zipimport.c abc $(IntDir)abc.g.h $(PySourcePath)Python\frozen_modules\abc.h - $(PySourcePath)Python\deepfreeze\abc.c + $(IntDir)abc.g.c + $(PySourcePath)Python\deepfreeze\df.abc.c codecs $(IntDir)codecs.g.h $(PySourcePath)Python\frozen_modules\codecs.h - $(PySourcePath)Python\deepfreeze\codecs_.c + $(IntDir)codecs.g.c + $(PySourcePath)Python\deepfreeze\df.codecs.c io $(IntDir)io.g.h $(PySourcePath)Python\frozen_modules\io.h - $(PySourcePath)Python\deepfreeze\io.c + $(IntDir)io.g.c + $(PySourcePath)Python\deepfreeze\df.io.c _collections_abc $(IntDir)_collections_abc.g.h $(PySourcePath)Python\frozen_modules\_collections_abc.h - $(PySourcePath)Python\deepfreeze\_collections_abc.c + $(IntDir)_collections_abc.g.c + $(PySourcePath)Python\deepfreeze\df._collections_abc.c _sitebuiltins $(IntDir)_sitebuiltins.g.h $(PySourcePath)Python\frozen_modules\_sitebuiltins.h - $(PySourcePath)Python\deepfreeze\_sitebuiltins.c + $(IntDir)_sitebuiltins.g.c + $(PySourcePath)Python\deepfreeze\df._sitebuiltins.c genericpath $(IntDir)genericpath.g.h $(PySourcePath)Python\frozen_modules\genericpath.h - $(PySourcePath)Python\deepfreeze\genericpath.c + $(IntDir)genericpath.g.c + $(PySourcePath)Python\deepfreeze\df.genericpath.c ntpath $(IntDir)ntpath.g.h $(PySourcePath)Python\frozen_modules\ntpath.h - $(PySourcePath)Python\deepfreeze\ntpath.c + $(IntDir)ntpath.g.c + $(PySourcePath)Python\deepfreeze\df.ntpath.c posixpath $(IntDir)posixpath.g.h $(PySourcePath)Python\frozen_modules\posixpath.h - $(PySourcePath)Python\deepfreeze\posixpath.c + $(IntDir)posixpath.g.c + $(PySourcePath)Python\deepfreeze\df.posixpath.c os $(IntDir)os.g.h $(PySourcePath)Python\frozen_modules\os.h - $(PySourcePath)Python\deepfreeze\os.c + $(IntDir)os.g.c + $(PySourcePath)Python\deepfreeze\df.os.c site $(IntDir)site.g.h $(PySourcePath)Python\frozen_modules\site.h - $(PySourcePath)Python\deepfreeze\site.c + $(IntDir)site.g.c + $(PySourcePath)Python\deepfreeze\df.site.c stat $(IntDir)stat.g.h $(PySourcePath)Python\frozen_modules\stat.h - $(PySourcePath)Python\deepfreeze\stat.c + $(IntDir)stat.g.c + $(PySourcePath)Python\deepfreeze\df.stat.c __hello__ $(IntDir)__hello__.g.h $(PySourcePath)Python\frozen_modules\__hello__.h - $(PySourcePath)Python\deepfreeze\__hello__.c + $(IntDir)__hello__.g.c + $(PySourcePath)Python\deepfreeze\df.__hello__.c __phello__ $(IntDir)__phello__.g.h $(PySourcePath)Python\frozen_modules\__phello__.h - $(PySourcePath)Python\deepfreeze\__phello__.c + $(IntDir)__phello__.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.c __phello__.ham $(IntDir)__phello__.ham.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.h - $(PySourcePath)Python\deepfreeze\__phello__.ham.c + $(IntDir)__phello__.ham.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.ham.c __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h - $(PySourcePath)Python\deepfreeze\__phello__.ham.eggs.c + $(IntDir)__phello__.ham.eggs.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.ham.eggs.c __phello__.spam $(IntDir)__phello__.spam.g.h $(PySourcePath)Python\frozen_modules\__phello__.spam.h - $(PySourcePath)Python\deepfreeze\__phello__.spam.c + $(IntDir)__phello__.spam.g.c + $(PySourcePath)Python\deepfreeze\df.__phello__.spam.c frozen_only $(IntDir)frozen_only.g.h $(PySourcePath)Python\frozen_modules\frozen_only.h - $(PySourcePath)Python\deepfreeze\frozen_only.c + $(IntDir)frozen_only.g.c + $(PySourcePath)Python\deepfreeze\df.frozen_only.c @@ -356,11 +376,11 @@ - + - + Condition="!Exists(%(None.OutFile)) or (Exists(%(None.DeepIntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(` `, ` `))' != '$([System.IO.File]::ReadAllText(%(None.DeepIntFile)).Replace(` `, ` `))')"> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 5cbcc35883ca4b..ece15606b5ed70 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -502,26 +502,26 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index be81c14a21a05c..61ccae61e4f965 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -721,15 +721,15 @@ def regen_pcbuild(modules): for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) - deepbase = src.id - if deepbase == "codecs": - deepbase = "codecs_" # Annoying exception (duplicate file) + deepbase = "df." + src.id deepoutfile = f"Python\\deepfreeze\\{deepbase}.c" intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h' + deepintfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.c' projlines.append(f' ') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') projlines.append(f' $(PySourcePath){header}') + projlines.append(f' $(IntDir){deepintfile}') projlines.append(f' $(PySourcePath){deepoutfile}') projlines.append(f' ') From f423c3bfeb4acc93f780d1ee1f6fcf896c70c51f Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sat, 20 Nov 2021 11:53:19 -0800 Subject: [PATCH 15/17] Remove unused imports from deepfreeze.py --- Tools/scripts/deepfreeze.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 7de8cb0d287bf3..b6d52b7454965f 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -3,10 +3,8 @@ import builtins import collections import contextlib -import marshal import os import re -import sys import time import types import typing From a3f60fcf0ec6942a0fd6ec991c0823ed05bf71ba Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 21 Nov 2021 10:46:48 -0800 Subject: [PATCH 16/17] Add a quick script to time startup --- Tools/scripts/startuptime.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 Tools/scripts/startuptime.py diff --git a/Tools/scripts/startuptime.py b/Tools/scripts/startuptime.py new file mode 100644 index 00000000000000..1bb5b208f66e04 --- /dev/null +++ b/Tools/scripts/startuptime.py @@ -0,0 +1,22 @@ +# Quick script to time startup for various binaries + +import subprocess +import sys +import time + +NREPS = 100 + + +def main(): + binaries = sys.argv[1:] + for bin in binaries: + t0 = time.time() + for _ in range(NREPS): + result = subprocess.run([bin, "-c", "pass"]) + result.check_returncode() + t1 = time.time() + print(f"{(t1-t0)/NREPS:6.3f} {bin}") + + +if __name__ == "__main__": + main() From a69895e798bd282a3b9b133d3e225b493f89b4fd Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 22 Nov 2021 08:51:41 -0800 Subject: [PATCH 17/17] Update blurb (reminder that 3.10 must be installed to build) --- .../next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst index 0bbb577a539159..a84e1feb750251 100644 --- a/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst +++ b/Misc/NEWS.d/next/Windows/2021-11-20-00-06-59.bpo-45850.q9lofz.rst @@ -1 +1,2 @@ -Implement changes to build with deep-frozen modules on Windows. \ No newline at end of file +Implement changes to build with deep-frozen modules on Windows. +Note that we now require Python 3.10 as the "bootstrap" or "host" Python.