From dcbde3a21ef7d9476181312c1effdf6206f3b4c1 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 17 Mar 2022 10:36:39 +0100 Subject: [PATCH 1/4] bpo-46890: Fix setting of sys._base_executable with framework builds on macOS The side effect of this bug was that venv environments directly used the main interpreter instead of the intermediate stub executable, which can cause problems when a script uses system APIs that require the use of an application bundle. --- Lib/test/test_getpath.py | 177 ++++++++++++++++++ Makefile.pre.in | 1 + .../2022-03-17-09-55-02.bpo-46890.GX-3OO.rst | 3 + Modules/getpath.c | 6 + Modules/getpath.py | 25 ++- 5 files changed, 207 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index eaf4a99279663e..b022cd24d5cc19 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -446,6 +446,182 @@ def test_custom_platlibdir_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_framework_macos(self): + """ Test framework layout on macOS + + This layout is primarily detected using a compile-time option + (WITH_NEXT_FRAMEWORK). + """ + ns = MockPosixNamespace( + os_name="darwin", + argv0="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python", + WITH_NEXT_FRAMEWORK=1, + PREFIX="/Library/Frameworks/Python.framework/Versions/9.8", + EXEC_PREFIX="/Library/Frameworks/Python.framework/Versions/9.8", + ENV___PYVENV_LAUNCHER__="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", + real_executable="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python", + library="/Library/Frameworks/Python.framework/Versions/9.8/Python", + ) + ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python") + ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8") + ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload") + ns.add_known_file("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py") + + # This is definitely not the stdlib (see discusion in bpo-46980) + #ns.add_known_file("/Library/Frameworks/lib/python98.zip") + + expected = dict( + executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", + prefix="/Library/Frameworks/Python.framework/Versions/9.8", + exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", + base_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + module_search_paths_set=1, + module_search_paths=[ + "/Library/Frameworks/Python.framework/Versions/9.8/lib/python98.zip", + "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8", + "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_alt_framework_macos(self): + """ Test framework layout on macOS with alternate framework name + + ``--with-framework-name=DebugPython`` + + This layout is primarily detected using a compile-time option + (WITH_NEXT_FRAMEWORK). + """ + ns = MockPosixNamespace( + argv0="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython", + os_name="darwin", + WITH_NEXT_FRAMEWORK=1, + PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8", + EXEC_PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8", + ENV___PYVENV_LAUNCHER__="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", + real_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython", + library="/Library/Frameworks/DebugPython.framework/Versions/9.8/DebugPython", + PYTHONPATH=None, + ENV_PYTHONHOME=None, + ENV_PYTHONEXECUTABLE=None, + executable_dir=None, + py_setpath=None, + ) + ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython") + ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8") + ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload") + ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py") + + # This is definitely not the stdlib (see discusion in bpo-46980) + #ns.add_known_xfile("/Library/lib/python98.zip") + expected = dict( + executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", + prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", + base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + module_search_paths_set=1, + module_search_paths=[ + "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python98.zip", + "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8", + "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_framework_macos(self): + """Test a venv layout on macOS using a framework build + """ + venv_path = "/tmp/workdir/venv" + ns = MockPosixNamespace( + os_name="darwin", + argv0="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python", + WITH_NEXT_FRAMEWORK=1, + PREFIX="/Library/Frameworks/Python.framework/Versions/9.8", + EXEC_PREFIX="/Library/Frameworks/Python.framework/Versions/9.8", + ENV___PYVENV_LAUNCHER__=f"{venv_path}/bin/python", + real_executable="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python", + library="/Library/Frameworks/Python.framework/Versions/9.8/Python", + ) + ns.add_known_dir(venv_path) + ns.add_known_dir(f"{venv_path}/bin") + ns.add_known_dir(f"{venv_path}/lib") + ns.add_known_dir(f"{venv_path}/lib/python9.8") + ns.add_known_xfile(f"{venv_path}/bin/python") + ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python") + ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8") + ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload") + ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py") + ns.add_known_file(f"{venv_path}/pyvenv.cfg", [ + "home = /Library/Frameworks/Python.framework/Versions/9.8/bin" + ]) + expected = dict( + executable=f"{venv_path}/bin/python", + prefix="/Library/Frameworks/Python.framework/Versions/9.8", + exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8", + base_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8", + module_search_paths_set=1, + module_search_paths=[ + "/Library/Frameworks/Python.framework/Versions/9.8/lib/python98.zip", + "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8", + "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + + def test_venv_alt_framework_macos(self): + """Test a venv layout on macOS using a framework build + + ``--with-framework-name=DebugPython`` + """ + venv_path = "/tmp/workdir/venv" + ns = MockPosixNamespace( + os_name="darwin", + argv0="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython", + WITH_NEXT_FRAMEWORK=1, + PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8", + EXEC_PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8", + ENV___PYVENV_LAUNCHER__=f"{venv_path}/bin/python", + real_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython", + library="/Library/Frameworks/DebugPython.framework/Versions/9.8/DebugPython", + ) + ns.add_known_dir(venv_path) + ns.add_known_dir(f"{venv_path}/bin") + ns.add_known_dir(f"{venv_path}/lib") + ns.add_known_dir(f"{venv_path}/lib/python9.8") + ns.add_known_xfile(f"{venv_path}/bin/python") + ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython") + ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8") + ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload") + ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py") + ns.add_known_file(f"{venv_path}/pyvenv.cfg", [ + "home = /Library/Frameworks/DebugPython.framework/Versions/9.8/bin" + ]) + expected = dict( + executable=f"{venv_path}/bin/python", + prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", + base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8", + module_search_paths_set=1, + module_search_paths=[ + "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python98.zip", + "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8", + "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload", + ], + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_macos(self): """Test a venv layout on macOS. @@ -787,6 +963,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "posix" self["PLATLIBDIR"] = "lib" + self["WITH_NEXT_FRAMEWORK"] = 0 super().__init__(*a, **kw) if argv0: self["config"]["orig_argv"] = [argv0] diff --git a/Makefile.pre.in b/Makefile.pre.in index c4034dc248c65b..0f9d4bbb4b4f3c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1221,6 +1221,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M -DVERSION='"$(VERSION)"' \ -DVPATH='"$(VPATH)"' \ -DPLATLIBDIR='"$(PLATLIBDIR)"' \ + -DPYTHONFRAMEWORK='"$(PYTHONFRAMEWORK)"' \ -o $@ $(srcdir)/Modules/getpath.c Programs/python.o: $(srcdir)/Programs/python.c diff --git a/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst b/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst new file mode 100644 index 00000000000000..1ab37c12349744 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst @@ -0,0 +1,3 @@ +Fix a regression in the setting of ``sys._base_exeutable`` in framework +builds, and thereby fix a regression in :mod:`venv` virtual environments +with such builds. diff --git a/Modules/getpath.c b/Modules/getpath.c index 5c646c9c83cbf4..890c537a8cc6fd 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -873,6 +873,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !decode_to_dict(dict, "os_name", "nt") || #elif defined(__APPLE__) !decode_to_dict(dict, "os_name", "darwin") || +#ifdef WITH_NEXT_FRAMEWORK + !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) || +#else + !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 0) || +#endif #else !decode_to_dict(dict, "os_name", "posix") || #endif @@ -943,3 +948,4 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) return _PyStatus_OK(); } + diff --git a/Modules/getpath.py b/Modules/getpath.py index 3a13bfdf491a14..4626fd85282ad5 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -33,6 +33,7 @@ # PREFIX -- [in] sysconfig.get_config_var(...) # EXEC_PREFIX -- [in] sysconfig.get_config_var(...) # PYTHONPATH -- [in] sysconfig.get_config_var(...) +# WITH_NEXT_FRAMEWORK -- [in] sysconfig.get_config_var(...) (on macOS only) # VPATH -- [in] sysconfig.get_config_var(...) # PLATLIBDIR -- [in] sysconfig.get_config_var(...) # PYDEBUGEXT -- [in, opt] '_d' on Windows for debug builds @@ -301,11 +302,25 @@ def search_up(prefix, *landmarks, test=isfile): # If set, these variables imply that we should be using them as # sys.executable and when searching for venvs. However, we should # use the argv0 path for prefix calculation - base_executable = executable - if not real_executable: - real_executable = executable - executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ - executable_dir = dirname(executable) + + if os_name == 'darwin' and WITH_NEXT_FRAMEWORK: + # In a framework build the binary in {sys.exec_prefix}/bin is + # a stub executable that execs the real interpreter in an + # embedded app bundle. That bundle is an implementation detail + # and should not affect base_execuble. + base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}" + if not real_executable: + real_executable = base_executable + real_executable_dir = dirname(real_executable) + executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ + executable_dir = dirname(executable) + + else: + base_executable = executable + if not real_executable: + real_executable = executable + executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ + executable_dir = dirname(executable) # ****************************************************************************** From 83e418c84a492580809154250d30cd1490605ace Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Thu, 17 Mar 2022 12:03:03 +0100 Subject: [PATCH 2/4] Fix patchcheck errors --- Lib/test/test_getpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index b022cd24d5cc19..4faa86b8906fcb 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -447,7 +447,7 @@ def test_custom_platlibdir_posix(self): self.assertEqual(expected, actual) def test_framework_macos(self): - """ Test framework layout on macOS + """ Test framework layout on macOS This layout is primarily detected using a compile-time option (WITH_NEXT_FRAMEWORK). From 353e7e7d3c86cd961ab0703de1214cd53ebc42ec Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Fri, 18 Mar 2022 14:25:22 +0100 Subject: [PATCH 3/4] Address review comments * Ensure WITH_NEXT_FRAMEWORK is set unconditionally while running getpath.py * Remove duplicate code in the WITH_NEXT_FRAMEWORK case. --- Modules/getpath.c | 6 +++--- Modules/getpath.py | 18 +++++++----------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Modules/getpath.c b/Modules/getpath.c index 890c537a8cc6fd..94479887cf850a 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -873,13 +873,13 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !decode_to_dict(dict, "os_name", "nt") || #elif defined(__APPLE__) !decode_to_dict(dict, "os_name", "darwin") || +#else + !decode_to_dict(dict, "os_name", "posix") || +#endif #ifdef WITH_NEXT_FRAMEWORK !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) || #else !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 0) || -#endif -#else - !decode_to_dict(dict, "os_name", "posix") || #endif !decode_to_dict(dict, "PREFIX", PREFIX) || !decode_to_dict(dict, "EXEC_PREFIX", EXEC_PREFIX) || diff --git a/Modules/getpath.py b/Modules/getpath.py index 4626fd85282ad5..1b4dcefe1fe842 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -33,7 +33,7 @@ # PREFIX -- [in] sysconfig.get_config_var(...) # EXEC_PREFIX -- [in] sysconfig.get_config_var(...) # PYTHONPATH -- [in] sysconfig.get_config_var(...) -# WITH_NEXT_FRAMEWORK -- [in] sysconfig.get_config_var(...) (on macOS only) +# WITH_NEXT_FRAMEWORK -- [in] sysconfig.get_config_var(...) # VPATH -- [in] sysconfig.get_config_var(...) # PLATLIBDIR -- [in] sysconfig.get_config_var(...) # PYDEBUGEXT -- [in, opt] '_d' on Windows for debug builds @@ -309,18 +309,14 @@ def search_up(prefix, *landmarks, test=isfile): # embedded app bundle. That bundle is an implementation detail # and should not affect base_execuble. base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}" - if not real_executable: - real_executable = base_executable - real_executable_dir = dirname(real_executable) - executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ - executable_dir = dirname(executable) - else: base_executable = executable - if not real_executable: - real_executable = executable - executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ - executable_dir = dirname(executable) + + if not real_executable: + real_executable = base_executable + #real_executable_dir = dirname(real_executable) + executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__ + executable_dir = dirname(executable) # ****************************************************************************** From bb7d6de13af00affb1ffe4991ef54558e26a71a1 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 5 Apr 2022 00:24:59 -0400 Subject: [PATCH 4/4] Fix minor typos in comments --- Lib/test/test_getpath.py | 4 ++-- .../next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst | 2 +- Modules/getpath.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 4faa86b8906fcb..5208374e20013c 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -467,7 +467,7 @@ def test_framework_macos(self): ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload") ns.add_known_file("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py") - # This is definitely not the stdlib (see discusion in bpo-46980) + # This is definitely not the stdlib (see discusion in bpo-46890) #ns.add_known_file("/Library/Frameworks/lib/python98.zip") expected = dict( @@ -515,7 +515,7 @@ def test_alt_framework_macos(self): ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload") ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py") - # This is definitely not the stdlib (see discusion in bpo-46980) + # This is definitely not the stdlib (see discusion in bpo-46890) #ns.add_known_xfile("/Library/lib/python98.zip") expected = dict( executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8", diff --git a/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst b/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst index 1ab37c12349744..a3d7d3e4ede021 100644 --- a/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst +++ b/Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst @@ -1,3 +1,3 @@ -Fix a regression in the setting of ``sys._base_exeutable`` in framework +Fix a regression in the setting of ``sys._base_executable`` in framework builds, and thereby fix a regression in :mod:`venv` virtual environments with such builds. diff --git a/Modules/getpath.py b/Modules/getpath.py index 1b4dcefe1fe842..26465c88aaea5c 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -307,7 +307,7 @@ def search_up(prefix, *landmarks, test=isfile): # In a framework build the binary in {sys.exec_prefix}/bin is # a stub executable that execs the real interpreter in an # embedded app bundle. That bundle is an implementation detail - # and should not affect base_execuble. + # and should not affect base_executable. base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}" else: base_executable = executable