From 4cc329b063e8e2f7a5b88c7841399466b607b8aa Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 2 Feb 2022 08:29:08 +0000 Subject: [PATCH 01/16] reduce size by removing frozen modules --- Python/frozen.c | 81 ++++++++++++--------------------- Python/import.c | 6 ++- Tools/scripts/freeze_modules.py | 34 ++++++++------ 3 files changed, 53 insertions(+), 68 deletions(-) diff --git a/Python/frozen.c b/Python/frozen.c index 25cf0a8d37c789..575fd80de62328 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -39,29 +39,6 @@ #include "pycore_import.h" /* Includes for frozen modules: */ -#include "frozen_modules/importlib._bootstrap.h" -#include "frozen_modules/importlib._bootstrap_external.h" -#include "frozen_modules/zipimport.h" -#include "frozen_modules/abc.h" -#include "frozen_modules/codecs.h" -#include "frozen_modules/io.h" -#include "frozen_modules/_collections_abc.h" -#include "frozen_modules/_sitebuiltins.h" -#include "frozen_modules/genericpath.h" -#include "frozen_modules/ntpath.h" -#include "frozen_modules/posixpath.h" -#include "frozen_modules/os.h" -#include "frozen_modules/site.h" -#include "frozen_modules/stat.h" -#include "frozen_modules/importlib.util.h" -#include "frozen_modules/importlib.machinery.h" -#include "frozen_modules/runpy.h" -#include "frozen_modules/__hello__.h" -#include "frozen_modules/__phello__.h" -#include "frozen_modules/__phello__.ham.h" -#include "frozen_modules/__phello__.ham.eggs.h" -#include "frozen_modules/__phello__.spam.h" -#include "frozen_modules/frozen_only.h" /* End includes */ #define GET_CODE(name) _Py_get_##name##_toplevel @@ -101,46 +78,46 @@ extern PyObject *_Py_get_frozen_only_toplevel(void); /* Note that a negative size indicates a package. */ static const struct _frozen bootstrap_modules[] = { - {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), GET_CODE(importlib__bootstrap)}, - {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), GET_CODE(importlib__bootstrap_external)}, - {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport), GET_CODE(zipimport)}, + {"_frozen_importlib", NULL, 0, GET_CODE(importlib__bootstrap)}, + {"_frozen_importlib_external", NULL, 0, GET_CODE(importlib__bootstrap_external)}, + {"zipimport", NULL, 0, GET_CODE(zipimport)}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ - {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), GET_CODE(abc)}, - {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, - {"io", _Py_M__io, (int)sizeof(_Py_M__io), GET_CODE(io)}, + {"abc", NULL, 0, GET_CODE(abc)}, + {"codecs", NULL, 0, GET_CODE(codecs)}, + {"io", NULL, 0, GET_CODE(io)}, /* stdlib - startup, with site */ - {"_collections_abc", _Py_M___collections_abc, (int)sizeof(_Py_M___collections_abc), GET_CODE(_collections_abc)}, - {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins), GET_CODE(_sitebuiltins)}, - {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath), GET_CODE(genericpath)}, - {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath), GET_CODE(ntpath)}, - {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, - {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath), GET_CODE(posixpath)}, - {"os", _Py_M__os, (int)sizeof(_Py_M__os), GET_CODE(os)}, - {"site", _Py_M__site, (int)sizeof(_Py_M__site), GET_CODE(site)}, - {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat), GET_CODE(stat)}, + {"_collections_abc", NULL, 0, GET_CODE(_collections_abc)}, + {"_sitebuiltins", NULL, 0, GET_CODE(_sitebuiltins)}, + {"genericpath", NULL, 0, GET_CODE(genericpath)}, + {"ntpath", NULL, 0, GET_CODE(ntpath)}, + {"posixpath", NULL, 0, GET_CODE(posixpath)}, + {"os.path", NULL, 0, GET_CODE(posixpath)}, + {"os", NULL, 0, GET_CODE(os)}, + {"site", NULL, 0, GET_CODE(site)}, + {"stat", NULL, 0, GET_CODE(stat)}, /* runpy - run module with -m */ - {"importlib.util", _Py_M__importlib_util, (int)sizeof(_Py_M__importlib_util), GET_CODE(importlib_util)}, - {"importlib.machinery", _Py_M__importlib_machinery, (int)sizeof(_Py_M__importlib_machinery), GET_CODE(importlib_machinery)}, - {"runpy", _Py_M__runpy, (int)sizeof(_Py_M__runpy), GET_CODE(runpy)}, + {"importlib.util", NULL, 0, GET_CODE(importlib_util)}, + {"importlib.machinery", NULL, 0, GET_CODE(importlib_machinery)}, + {"runpy", NULL, 0, GET_CODE(runpy)}, {0, 0, 0} /* stdlib sentinel */ }; static const struct _frozen test_modules[] = { - {"__hello__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, - {"__hello_alias__", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, - {"__phello_alias__", _Py_M____hello__, -(int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, - {"__phello_alias__.spam", _Py_M____hello__, (int)sizeof(_Py_M____hello__), GET_CODE(__hello__)}, - {"__phello__", _Py_M____phello__, -(int)sizeof(_Py_M____phello__), GET_CODE(__phello__)}, - {"__phello__.__init__", _Py_M____phello__, (int)sizeof(_Py_M____phello__), GET_CODE(__phello__)}, - {"__phello__.ham", _Py_M____phello___ham, -(int)sizeof(_Py_M____phello___ham), GET_CODE(__phello___ham)}, - {"__phello__.ham.__init__", _Py_M____phello___ham, (int)sizeof(_Py_M____phello___ham), GET_CODE(__phello___ham)}, - {"__phello__.ham.eggs", _Py_M____phello___ham_eggs, (int)sizeof(_Py_M____phello___ham_eggs), GET_CODE(__phello___ham_eggs)}, - {"__phello__.spam", _Py_M____phello___spam, (int)sizeof(_Py_M____phello___spam), GET_CODE(__phello___spam)}, - {"__hello_only__", _Py_M__frozen_only, (int)sizeof(_Py_M__frozen_only), GET_CODE(frozen_only)}, + {"__hello__", NULL, 0, GET_CODE(__hello__)}, + {"__hello_alias__", NULL, 0, GET_CODE(__hello__)}, + {"__phello_alias__", NULL, 0, GET_CODE(__hello__)}, + {"__phello_alias__.spam", NULL, 0, GET_CODE(__hello__)}, + {"__phello__", NULL, 0, GET_CODE(__phello__)}, + {"__phello__.__init__", NULL, 0, GET_CODE(__phello__)}, + {"__phello__.ham", NULL, 0, GET_CODE(__phello___ham)}, + {"__phello__.ham.__init__", NULL, 0, GET_CODE(__phello___ham)}, + {"__phello__.ham.eggs", NULL, 0, GET_CODE(__phello___ham_eggs)}, + {"__phello__.spam", NULL, 0, GET_CODE(__phello___spam)}, + {"__hello_only__", NULL, 0, GET_CODE(frozen_only)}, {0, 0, 0} /* test sentinel */ }; const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules; diff --git a/Python/import.c b/Python/import.c index 51b779ca17c526..7a703eb9d2abf8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1303,7 +1303,9 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, &info->origname); } - + if (p->code == NULL && p->size == 0 && p->get_code != NULL) { + return FROZEN_OKAY; + } if (p->code == NULL) { /* It is frozen but marked as un-importable. */ return FROZEN_EXCLUDED; @@ -2224,7 +2226,7 @@ _imp_get_frozen_object_impl(PyObject *module, PyObject *name, if (info.nameobj == NULL) { info.nameobj = name; } - if (info.size == 0) { + if (info.size == 0 && info.get_code == NULL) { /* Does not contain executable code. */ set_frozen_error(FROZEN_INVALID, name); return NULL; diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 6d10758b5285cc..6849e6f3e6d9d9 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -9,7 +9,7 @@ import ntpath import posixpath import sys - +import argparse from update_file import updating_file_with_tmpfile @@ -463,14 +463,15 @@ def replace_block(lines, start_marker, end_marker, replacements, file): return lines[:start_pos + 1] + replacements + lines[end_pos:] -def regen_frozen(modules): +def regen_frozen(modules, deepfreeze_only: bool): headerlines = [] parentdir = os.path.dirname(FROZEN_FILE) - for src in _iter_sources(modules): - # Adding a comment to separate sections here doesn't add much, - # so we don't. - header = relpath_for_posix_display(src.frozenfile, parentdir) - headerlines.append(f'#include "{header}"') + if not deepfreeze_only: + for src in _iter_sources(modules): + # Adding a comment to separate sections here doesn't add much, + # so we don't. + header = relpath_for_posix_display(src.frozenfile, parentdir) + headerlines.append(f'#include "{header}"') externlines = [] bootstraplines = [] @@ -501,7 +502,11 @@ def regen_frozen(modules): symbol = mod.symbol pkg = '-' if mod.ispkg else '' - line = ('{"%s", %s, %s(int)sizeof(%s), GET_CODE(%s)},' + if deepfreeze_only: + line = ('{"%s", NULL, 0, GET_CODE(%s)},' + ) % (mod.name, code_name) + else: + line = ('{"%s", %s, %s(int)sizeof(%s), GET_CODE(%s)},' ) % (mod.name, symbol, pkg, symbol, code_name) lines.append(line) @@ -710,18 +715,19 @@ def regen_pcbuild(modules): ####################################### # the script -def main(): +def main(deepfreeze_only: bool): # Expand the raw specs, preserving order. modules = list(parse_frozen_specs()) # Regen build-related files. regen_makefile(modules) regen_pcbuild(modules) - regen_frozen(modules) + regen_frozen(modules, deepfreeze_only) if __name__ == '__main__': - argv = sys.argv[1:] - if argv: - sys.exit(f'ERROR: got unexpected args {argv}') - main() + parser = argparse.ArgumentParser() + parser.add_argument("--deepfreeze-only", action="store_true", + help="Only use deepfrozen modules", default=True) + args = parser.parse_args() + main(args.deepfreeze_only) From 1bc552b489032e1b2448a5db4659bdbbe8d401fb Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 2 Feb 2022 08:43:51 +0000 Subject: [PATCH 02/16] add comment --- Python/import.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/import.c b/Python/import.c index 7a703eb9d2abf8..a72d293a0515c7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1304,6 +1304,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) &info->origname); } if (p->code == NULL && p->size == 0 && p->get_code != NULL) { + /* It is only deepfrozen. */ return FROZEN_OKAY; } if (p->code == NULL) { From 0626e51bae110b334cf522142ff7dce26178502f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:44:35 +0000 Subject: [PATCH 03/16] refactor --- Include/cpython/import.h | 1 + Lib/ctypes/test/test_values.py | 10 +++--- Python/frozen.c | 58 ++++++++++++++++----------------- Python/import.c | 4 +-- Tools/scripts/freeze_modules.py | 10 +++--- 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 5ec637e7ab3b8c..da9fb770a92453 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -32,6 +32,7 @@ struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; int size; + bool is_package; PyObject *(*get_code)(void); }; diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py index 3e8b13768b421a..b2db426d10f431 100644 --- a/Lib/ctypes/test/test_values.py +++ b/Lib/ctypes/test/test_values.py @@ -54,6 +54,7 @@ class struct_frozen(Structure): _fields_ = [("name", c_char_p), ("code", POINTER(c_ubyte)), ("size", c_int), + ("is_package", c_bool), ("get_code", POINTER(c_ubyte)), # Function ptr ] FrozenTable = POINTER(struct_frozen) @@ -71,13 +72,14 @@ class struct_frozen(Structure): modname = entry.name.decode("ascii") modules.append(modname) with self.subTest(modname): - # Do a sanity check on entry.size and entry.code. - self.assertGreater(abs(entry.size), 10) - self.assertTrue([entry.code[i] for i in range(abs(entry.size))]) + if entry.size != 0: + # Do a sanity check on entry.size and entry.code. + self.assertGreater(abs(entry.size), 10) + self.assertTrue([entry.code[i] for i in range(abs(entry.size))]) # Check the module's package-ness. with import_helper.frozen_modules(): spec = importlib.util.find_spec(modname) - if entry.size < 0: + if entry.is_package: # It's a package. self.assertIsNotNone(spec.submodule_search_locations) else: diff --git a/Python/frozen.c b/Python/frozen.c index 575fd80de62328..0c20ff2ff32b4e 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -78,46 +78,46 @@ extern PyObject *_Py_get_frozen_only_toplevel(void); /* Note that a negative size indicates a package. */ static const struct _frozen bootstrap_modules[] = { - {"_frozen_importlib", NULL, 0, GET_CODE(importlib__bootstrap)}, - {"_frozen_importlib_external", NULL, 0, GET_CODE(importlib__bootstrap_external)}, - {"zipimport", NULL, 0, GET_CODE(zipimport)}, + {"_frozen_importlib", NULL, 0, false, GET_CODE(importlib__bootstrap)}, + {"_frozen_importlib_external", NULL, 0, false, GET_CODE(importlib__bootstrap_external)}, + {"zipimport", NULL, 0, false, GET_CODE(zipimport)}, {0, 0, 0} /* bootstrap sentinel */ }; static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ - {"abc", NULL, 0, GET_CODE(abc)}, - {"codecs", NULL, 0, GET_CODE(codecs)}, - {"io", NULL, 0, GET_CODE(io)}, + {"abc", NULL, 0, false, GET_CODE(abc)}, + {"codecs", NULL, 0, false, GET_CODE(codecs)}, + {"io", NULL, 0, false, GET_CODE(io)}, /* stdlib - startup, with site */ - {"_collections_abc", NULL, 0, GET_CODE(_collections_abc)}, - {"_sitebuiltins", NULL, 0, GET_CODE(_sitebuiltins)}, - {"genericpath", NULL, 0, GET_CODE(genericpath)}, - {"ntpath", NULL, 0, GET_CODE(ntpath)}, - {"posixpath", NULL, 0, GET_CODE(posixpath)}, - {"os.path", NULL, 0, GET_CODE(posixpath)}, - {"os", NULL, 0, GET_CODE(os)}, - {"site", NULL, 0, GET_CODE(site)}, - {"stat", NULL, 0, GET_CODE(stat)}, + {"_collections_abc", NULL, 0, false, GET_CODE(_collections_abc)}, + {"_sitebuiltins", NULL, 0, false, GET_CODE(_sitebuiltins)}, + {"genericpath", NULL, 0, false, GET_CODE(genericpath)}, + {"ntpath", NULL, 0, false, GET_CODE(ntpath)}, + {"posixpath", NULL, 0, false, GET_CODE(posixpath)}, + {"os.path", NULL, 0, false, GET_CODE(posixpath)}, + {"os", NULL, 0, false, GET_CODE(os)}, + {"site", NULL, 0, false, GET_CODE(site)}, + {"stat", NULL, 0, false, GET_CODE(stat)}, /* runpy - run module with -m */ - {"importlib.util", NULL, 0, GET_CODE(importlib_util)}, - {"importlib.machinery", NULL, 0, GET_CODE(importlib_machinery)}, - {"runpy", NULL, 0, GET_CODE(runpy)}, + {"importlib.util", NULL, 0, false, GET_CODE(importlib_util)}, + {"importlib.machinery", NULL, 0, false, GET_CODE(importlib_machinery)}, + {"runpy", NULL, 0, false, GET_CODE(runpy)}, {0, 0, 0} /* stdlib sentinel */ }; static const struct _frozen test_modules[] = { - {"__hello__", NULL, 0, GET_CODE(__hello__)}, - {"__hello_alias__", NULL, 0, GET_CODE(__hello__)}, - {"__phello_alias__", NULL, 0, GET_CODE(__hello__)}, - {"__phello_alias__.spam", NULL, 0, GET_CODE(__hello__)}, - {"__phello__", NULL, 0, GET_CODE(__phello__)}, - {"__phello__.__init__", NULL, 0, GET_CODE(__phello__)}, - {"__phello__.ham", NULL, 0, GET_CODE(__phello___ham)}, - {"__phello__.ham.__init__", NULL, 0, GET_CODE(__phello___ham)}, - {"__phello__.ham.eggs", NULL, 0, GET_CODE(__phello___ham_eggs)}, - {"__phello__.spam", NULL, 0, GET_CODE(__phello___spam)}, - {"__hello_only__", NULL, 0, GET_CODE(frozen_only)}, + {"__hello__", NULL, 0, false, GET_CODE(__hello__)}, + {"__hello_alias__", NULL, 0, false, GET_CODE(__hello__)}, + {"__phello_alias__", NULL, 0, true, GET_CODE(__hello__)}, + {"__phello_alias__.spam", NULL, 0, false, GET_CODE(__hello__)}, + {"__phello__", NULL, 0, true, GET_CODE(__phello__)}, + {"__phello__.__init__", NULL, 0, false, GET_CODE(__phello__)}, + {"__phello__.ham", NULL, 0, true, GET_CODE(__phello___ham)}, + {"__phello__.ham.__init__", NULL, 0, false, GET_CODE(__phello___ham)}, + {"__phello__.ham.eggs", NULL, 0, false, GET_CODE(__phello___ham_eggs)}, + {"__phello__.spam", NULL, 0, false, GET_CODE(__phello___spam)}, + {"__hello_only__", NULL, 0, false, GET_CODE(frozen_only)}, {0, 0, 0} /* test sentinel */ }; const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules; diff --git a/Python/import.c b/Python/import.c index a72d293a0515c7..8b75eceff8458d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1297,8 +1297,8 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) info->nameobj = nameobj; // borrowed info->data = (const char *)p->code; info->get_code = p->get_code; - info->size = p->size < 0 ? -(p->size) : p->size; - info->is_package = p->size < 0 ? true : false; + info->size = p->size; + info->is_package = p->is_package; info->origname = name; info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, &info->origname); diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 6849e6f3e6d9d9..03dcf939f978ef 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -501,13 +501,13 @@ def regen_frozen(modules, deepfreeze_only: bool): externlines.append("extern PyObject *%s(void);" % get_code_name) symbol = mod.symbol - pkg = '-' if mod.ispkg else '' + pkg = 'true' if mod.ispkg else 'false' if deepfreeze_only: - line = ('{"%s", NULL, 0, GET_CODE(%s)},' - ) % (mod.name, code_name) + line = ('{"%s", NULL, 0, %s, GET_CODE(%s)},' + ) % (mod.name, pkg, code_name) else: - line = ('{"%s", %s, %s(int)sizeof(%s), GET_CODE(%s)},' - ) % (mod.name, symbol, pkg, symbol, code_name) + line = ('{"%s", %s, (int)sizeof(%s), %s, GET_CODE(%s)},' + ) % (mod.name, symbol, symbol, pkg, code_name) lines.append(line) if mod.isalias: From 259e8c0a3202885091e9a2b1dbc7c6139e2c1a8d Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 2 Feb 2022 09:45:50 +0000 Subject: [PATCH 04/16] remove outdated comment --- Python/frozen.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/frozen.c b/Python/frozen.c index 0c20ff2ff32b4e..c5b36f73b4a473 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -75,8 +75,6 @@ extern PyObject *_Py_get___phello___spam_toplevel(void); extern PyObject *_Py_get_frozen_only_toplevel(void); /* End extern declarations */ -/* Note that a negative size indicates a package. */ - static const struct _frozen bootstrap_modules[] = { {"_frozen_importlib", NULL, 0, false, GET_CODE(importlib__bootstrap)}, {"_frozen_importlib_external", NULL, 0, false, GET_CODE(importlib__bootstrap_external)}, From 01592e7a48d8f2e2aa7eb48a40e72602ae8ae736 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 2 Feb 2022 11:25:18 +0000 Subject: [PATCH 05/16] fix tests --- Tools/freeze/makefreeze.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/freeze/makefreeze.py b/Tools/freeze/makefreeze.py index d7d05db88a96c9..bc5f8567448bfd 100644 --- a/Tools/freeze/makefreeze.py +++ b/Tools/freeze/makefreeze.py @@ -45,19 +45,19 @@ def makefreeze(base, dict, debug=0, entry_point=None, fail_import=()): print("freezing", mod, "...") str = marshal.dumps(m.__code__) size = len(str) + is_package = 'false' if m.__path__: - # Indicate package by negative size - size = -size - done.append((mod, mangled, size)) + is_package = 'true' + done.append((mod, mangled, size, is_package)) writecode(outfp, mangled, str) if debug: print("generating table of frozen modules") with bkfile.open(base + 'frozen.c', 'w') as outfp: - for mod, mangled, size in done: + for mod, mangled, size, _ in done: outfp.write('extern unsigned char M_%s[];\n' % mangled) outfp.write(header) - for mod, mangled, size in done: - outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mangled, size)) + for mod, mangled, size, is_package in done: + outfp.write('\t{"%s", M_%s, %d, %s},\n' % (mod, mangled, size, is_package)) outfp.write('\n') # The following modules have a NULL code pointer, indicating # that the frozen program should not search for them on the host From f1c155b7b6ee01f9d22c8cefd4ee2d5cf8e1bfd0 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 2 Feb 2022 11:26:47 +0000 Subject: [PATCH 06/16] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst diff --git a/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst b/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst new file mode 100644 index 00000000000000..6085e4ec31e1c6 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst @@ -0,0 +1 @@ +Exclude marshalled-frozen data if deep-freezing to save 300 KB disk space. Patch by Kumar Aditya. \ No newline at end of file From a9cbe0fc0233190c4f9dc274f5373cb810293609 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 3 Feb 2022 09:26:16 +0530 Subject: [PATCH 07/16] add backwards compatibility --- Python/import.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/import.c b/Python/import.c index 8b75eceff8458d..4d764b0086791d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1299,6 +1299,11 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) info->get_code = p->get_code; info->size = p->size; info->is_package = p->is_package; + if (p->size < 0 ) { + // backward compatibility with negative size values + info->size = -(p->size); + info->is_package = true; + } info->origname = name; info->is_alias = resolve_module_alias(name, _PyImport_FrozenAliases, &info->origname); From cb1ad2cb2fc218f9dddb7478aff0df425b9c13e4 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 3 Feb 2022 09:59:59 +0530 Subject: [PATCH 08/16] fix style --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 4d764b0086791d..be60c431f7ef8f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1299,7 +1299,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) info->get_code = p->get_code; info->size = p->size; info->is_package = p->is_package; - if (p->size < 0 ) { + if (p->size < 0) { // backward compatibility with negative size values info->size = -(p->size); info->is_package = true; From 1e43d8bd116e1ceade9e1cab64025a301f7696f5 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 09:12:01 +0530 Subject: [PATCH 09/16] add docs --- Doc/c-api/import.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index d2ae6b6d4e4711..fb9c14a0b1b80d 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -256,8 +256,12 @@ Importing Modules const char *name; const unsigned char *code; int size; + bool is_package; }; + .. versionchanged:: 3.11 + The ``is_package`` is used to indicate whether the module is a package instead of + negating the value of the ``size`` field. .. c:var:: const struct _frozen* PyImport_FrozenModules From 5927eebb09ebee271a484ee5f219dbb3ad96e594 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 09:13:12 +0530 Subject: [PATCH 10/16] reword --- Doc/c-api/import.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index fb9c14a0b1b80d..f6992a113aaf7b 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -260,8 +260,8 @@ Importing Modules }; .. versionchanged:: 3.11 - The ``is_package`` is used to indicate whether the module is a package instead of - negating the value of the ``size`` field. + The ``is_package`` field is used to indicate whether the module is a package + instead of negating the value of the ``size`` field. .. c:var:: const struct _frozen* PyImport_FrozenModules From 1f26dced797d759530eba000ee6c77f4274c4574 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 09:14:35 +0530 Subject: [PATCH 11/16] added --- Doc/c-api/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index f6992a113aaf7b..40ca5d6ca1f326 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -260,7 +260,7 @@ Importing Modules }; .. versionchanged:: 3.11 - The ``is_package`` field is used to indicate whether the module is a package + Added ``is_package`` field is used to indicate whether the module is a package instead of negating the value of the ``size`` field. .. c:var:: const struct _frozen* PyImport_FrozenModules From 233d65d4e63927fa2a4015b3cb7f187361f01d92 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 09:20:45 +0530 Subject: [PATCH 12/16] added whatsnew --- Doc/whatsnew/3.11.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index e7f3dab2b51db4..efc700cc15df49 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -638,6 +638,10 @@ C API Changes fields of the result from the exception instance (the ``value`` field). (Contributed by Irit Katriel in :issue:`45711`.) +* :c:struct:`_frozen` has a new ``is_package`` field that indicates whether + the frozen module is a package instead of negating the value of ``size`` + field. + (Contributed by Kumar Aditya in :issue:`46608`.) New Features ------------ From 04ad81375ce9f2e58352b5559a91dcdf824ef706 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 22:52:17 +0530 Subject: [PATCH 13/16] Update Doc/c-api/import.rst Co-authored-by: Eric Snow --- Doc/c-api/import.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 40ca5d6ca1f326..5e2333a74ce648 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -260,8 +260,8 @@ Importing Modules }; .. versionchanged:: 3.11 - Added ``is_package`` field is used to indicate whether the module is a package - instead of negating the value of the ``size`` field. + The new ``is_package`` field indicates whether the module is a package or not. + This replaces setting the ``size`` field to a negative value. .. c:var:: const struct _frozen* PyImport_FrozenModules From 2587822dfe134cdf96170468ec55ab192ce083a8 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 22:52:24 +0530 Subject: [PATCH 14/16] Update Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst Co-authored-by: Eric Snow --- .../NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst b/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst index 6085e4ec31e1c6..335bb6a5a8d53f 100644 --- a/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst +++ b/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst @@ -1 +1,2 @@ -Exclude marshalled-frozen data if deep-freezing to save 300 KB disk space. Patch by Kumar Aditya. \ No newline at end of file +Exclude marshalled-frozen data if deep-freezing to save 300 KB disk space. This includes adding +a new ``is_package`` field to :cstruct:`_frozen`. Patch by Kumar Aditya. \ No newline at end of file From 3a593caba5cb4545d3ba0026ed6b19da88a45c3e Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 22:52:33 +0530 Subject: [PATCH 15/16] Update Doc/whatsnew/3.11.rst Co-authored-by: Eric Snow --- Doc/whatsnew/3.11.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index efc700cc15df49..ec0cb21936f382 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -638,9 +638,10 @@ C API Changes fields of the result from the exception instance (the ``value`` field). (Contributed by Irit Katriel in :issue:`45711`.) -* :c:struct:`_frozen` has a new ``is_package`` field that indicates whether - the frozen module is a package instead of negating the value of ``size`` - field. +* :c:struct:`_frozen` has a new ``is_package`` field to indicate whether + or not the frozen module is a package. Previously, a negative value + in the ``size`` field was the indicator. Now only non-negative values + be used for ``size``. (Contributed by Kumar Aditya in :issue:`46608`.) New Features From e4feaa20abb1670cfd63e8d2e4203d5ca0789ac2 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 4 Feb 2022 23:03:57 +0530 Subject: [PATCH 16/16] Update 2022-02-02-11-26-46.bpo-46608.cXH9po.rst --- Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst b/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst index 335bb6a5a8d53f..13c73a614e5e86 100644 --- a/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst +++ b/Misc/NEWS.d/next/Build/2022-02-02-11-26-46.bpo-46608.cXH9po.rst @@ -1,2 +1,2 @@ Exclude marshalled-frozen data if deep-freezing to save 300 KB disk space. This includes adding -a new ``is_package`` field to :cstruct:`_frozen`. Patch by Kumar Aditya. \ No newline at end of file +a new ``is_package`` field to :c:struct:`_frozen`. Patch by Kumar Aditya.