From ef7b0050da02f3aee45ab1b8880288b431d601db Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 26 Jun 2019 16:53:10 -0700 Subject: [PATCH 01/16] [WIP] Attempt to use alternate executable path --- Include/cpython/initconfig.h | 9 +- PC/python_uwp.cpp | 198 ++++++++++++++++------------------- Python/initconfig.c | 4 + Python/pathconfig.c | 6 ++ Python/sysmodule.c | 1 + 5 files changed, 104 insertions(+), 114 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 67f38e26505c2a..5fb6fade553731 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -373,10 +373,11 @@ typedef struct { module_search_paths_set is equal to zero. */ - wchar_t *executable; /* sys.executable */ - wchar_t *prefix; /* sys.prefix */ - wchar_t *base_prefix; /* sys.base_prefix */ - wchar_t *exec_prefix; /* sys.exec_prefix */ + wchar_t *executable; /* sys.executable */ + wchar_t *base_executable; /* sys.base_executable */ + wchar_t *prefix; /* sys.prefix */ + wchar_t *base_prefix; /* sys.base_prefix */ + wchar_t *exec_prefix; /* sys.exec_prefix */ wchar_t *base_exec_prefix; /* sys.base_exec_prefix */ /* --- Parameter only used by Py_Main() ---------- */ diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index dd1edde730921b..5ae5a123ae5521 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -46,91 +46,48 @@ set_user_base() } } -static const wchar_t * -get_argv0(const wchar_t *argv0) +static winrt::hstring +get_package_family() { - winrt::hstring installPath; - const wchar_t *launcherPath; - wchar_t *buffer; - size_t len; - - launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); - if (launcherPath && launcherPath[0]) { - len = wcslen(launcherPath) + 1; - buffer = (wchar_t *)malloc(sizeof(wchar_t) * len); - if (!buffer) { - Py_FatalError("out of memory"); - return NULL; - } - if (wcscpy_s(buffer, len, launcherPath)) { - Py_FatalError("failed to copy to buffer"); - return NULL; - } - return buffer; - } - try { const auto package = winrt::Windows::ApplicationModel::Package::Current(); if (package) { - const auto install = package.InstalledLocation(); - if (install) { - installPath = install.Path(); - } + const auto id = package.Id(); + return id ? id.FamilyName() : winrt::hstring(); } } catch (...) { } - if (!installPath.empty()) { - len = installPath.size() + wcslen(PROGNAME) + 2; - } else { - len = wcslen(argv0) + wcslen(PROGNAME) + 1; - } - - buffer = (wchar_t *)malloc(sizeof(wchar_t) * len); - if (!buffer) { - Py_FatalError("out of memory"); - return NULL; - } - - if (!installPath.empty()) { - if (wcscpy_s(buffer, len, installPath.c_str())) { - Py_FatalError("failed to copy to buffer"); - return NULL; - } - if (wcscat_s(buffer, len, L"\\")) { - Py_FatalError("failed to concatenate backslash"); - return NULL; - } - } else { - if (wcscpy_s(buffer, len, argv0)) { - Py_FatalError("failed to copy argv[0]"); - return NULL; - } - - wchar_t *name = wcsrchr(buffer, L'\\'); - if (name) { - name[1] = L'\0'; - } else { - buffer[0] = L'\0'; - } - } - - if (wcscat_s(buffer, len, PROGNAME)) { - Py_FatalError("failed to concatenate program name"); - return NULL; - } - - return buffer; + return winrt::hstring(); } -static wchar_t * -get_process_name() +static int +set_process_name(PyConfig *config) { DWORD bufferLen = MAX_PATH; DWORD len = bufferLen; wchar_t *r = NULL; + const auto family = get_package_family(); + while (!family.empty() && !r) { + r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); + if (!r) { + Py_FatalError("out of memory"); + return 0; + } + len = _snwprintf_s(r, bufferLen, _TRUNCATE, + L"%ls\\Microsoft\\WindowsApps\\%ls\\%ls", + _wgetenv(L"LOCALAPPDATA"), + family.c_str(), + PROGNAME); + if (len < 0) { + free((void *)r); + r = NULL; + bufferLen *= 2; + } + } + while (!r) { r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); if (!r) { @@ -140,7 +97,7 @@ get_process_name() len = GetModuleFileNameW(NULL, r, bufferLen); if (len == 0) { free((void *)r); - return NULL; + return 0; } else if (len == bufferLen && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { free(r); @@ -149,67 +106,88 @@ get_process_name() } } - return r; + PyConfig_SetString(config, &config->base_executable, r); + const wchar_t *launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); + if (launcherPath) { + PyConfig_SetString(config, &config->executable, launcherPath); + } else { + PyConfig_SetString(config, &config->executable, r); + } + free((void *)r); + + return 1; } int wmain(int argc, wchar_t **argv) { - const wchar_t **new_argv; - int new_argc; - const wchar_t *exeName; - - new_argc = argc; - new_argv = (const wchar_t**)malloc(sizeof(wchar_t *) * (argc + 2)); - if (new_argv == NULL) { - Py_FatalError("out of memory"); - return -1; - } + PyStatus status; - exeName = get_process_name(); + PyPreConfig preconfig; + PyConfig config; - new_argv[0] = get_argv0(exeName ? exeName : argv[0]); - for (int i = 1; i < argc; ++i) { - new_argv[i] = argv[i]; + PyPreConfig_InitPythonConfig(&preconfig); + status = Py_PreInitializeFromArgs(&preconfig, argc, argv); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = PyConfig_InitPythonConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + + status = PyConfig_SetArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto fail; } + if (!set_process_name(&config)) { + status = PyStatus_Exit(121); + goto fail; + } set_user_base(); - if (exeName) { - const wchar_t *p = wcsrchr(exeName, L'\\'); - if (p) { + const wchar_t *p = wcsrchr(argv[0], L'\\'); + if (!p) { + p = argv[0]; + } + if (p) { + if (*p++ == L'\\') { const wchar_t *moduleName = NULL; - if (*p++ == L'\\') { - if (wcsnicmp(p, L"pip", 3) == 0) { - moduleName = L"pip"; - /* No longer required when pip 19.1 is added */ - _wputenv_s(L"PIP_USER", L"true"); - } else if (wcsnicmp(p, L"idle", 4) == 0) { - moduleName = L"idlelib"; - } + if (wcsnicmp(p, L"pip", 3) == 0) { + moduleName = L"pip"; + /* No longer required when pip 19.1 is added */ + _wputenv_s(L"PIP_USER", L"true"); + } else if (wcsnicmp(p, L"idle", 4) == 0) { + moduleName = L"idlelib"; } if (moduleName) { - new_argc += 2; - for (int i = argc; i >= 1; --i) { - new_argv[i + 2] = new_argv[i]; - } - new_argv[1] = L"-m"; - new_argv[2] = moduleName; + PyConfig_SetString(&config, &config.run_module, moduleName); + PyConfig_SetString(&config, &config.run_filename, NULL); + PyConfig_SetString(&config, &config.run_command, NULL); } } } - /* Override program_full_path from here so that - sys.executable is set correctly. */ - _Py_SetProgramFullPath(new_argv[0]); - - int result = Py_Main(new_argc, (wchar_t **)new_argv); + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto fail; + } + PyConfig_Clear(&config); - free((void *)exeName); - free((void *)new_argv); + return Py_RunMain(); - return result; +fail: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + assert(PyStatus_Exception(status)); + Py_ExitStatusException(status); + /* Unreachable code */ + return 0; } #ifdef PYTHONW diff --git a/Python/initconfig.c b/Python/initconfig.c index 9c4cfbeb6b1b8b..786f6945c17149 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -528,6 +528,7 @@ PyConfig_Clear(PyConfig *config) config->module_search_paths_set = 0; CLEAR(config->executable); + CLEAR(config->base_executable); CLEAR(config->prefix); CLEAR(config->base_prefix); CLEAR(config->exec_prefix); @@ -765,6 +766,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_ATTR(module_search_paths_set); COPY_WSTR_ATTR(executable); + COPY_WSTR_ATTR(base_executable); COPY_WSTR_ATTR(prefix); COPY_WSTR_ATTR(base_prefix); COPY_WSTR_ATTR(exec_prefix); @@ -865,6 +867,7 @@ config_as_dict(const PyConfig *config) SET_ITEM_WSTR(home); SET_ITEM_WSTRLIST(module_search_paths); SET_ITEM_WSTR(executable); + SET_ITEM_WSTR(base_executable); SET_ITEM_WSTR(prefix); SET_ITEM_WSTR(base_prefix); SET_ITEM_WSTR(exec_prefix); @@ -2404,6 +2407,7 @@ PyConfig_Read(PyConfig *config) assert(config->module_search_paths_set != 0); /* don't check config->module_search_paths */ assert(config->executable != NULL); + assert(config->base_executable != NULL); assert(config->prefix != NULL); assert(config->base_prefix != NULL); assert(config->exec_prefix != NULL); diff --git a/Python/pathconfig.c b/Python/pathconfig.c index ec67405a28d054..fea397cca22090 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -355,6 +355,12 @@ _PyConfig_InitPathConfig(PyConfig *config) } } + if (config->base_executable == NULL) { + if (copy_wstr(&config->base_executable, config->executable) < 0) { + return _PyStatus_NO_MEMORY(); + } + } + if (config->base_prefix == NULL) { if (copy_wstr(&config->base_prefix, config->prefix) < 0) { return _PyStatus_NO_MEMORY(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b200318c75f2c9..aa5834aac366d1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2850,6 +2850,7 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyThreadState *tstate) COPY_LIST("path", config->module_search_paths); SET_SYS_FROM_WSTR("executable", config->executable); + SET_SYS_FROM_WSTR("base_executable", config->base_executable); SET_SYS_FROM_WSTR("prefix", config->prefix); SET_SYS_FROM_WSTR("base_prefix", config->base_prefix); SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix); From 00db87b9e70fd4f530b1ada75c47f1a99b6431c5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 08:38:33 -0700 Subject: [PATCH 02/16] [WIP] Finish sys.base_executable changes, improve alias path calculation --- Lib/multiprocessing/popen_spawn_win32.py | 5 ++-- Lib/test/test_venv.py | 15 ++-------- Lib/venv/__init__.py | 2 +- PC/python_uwp.cpp | 37 +++++++++++++----------- 4 files changed, 25 insertions(+), 34 deletions(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index de4c5ecf1fa083..136b8b3a96e96a 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -22,8 +22,7 @@ def _path_eq(p1, p2): return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2) -WINENV = (hasattr(sys, '_base_executable') and - not _path_eq(sys.executable, sys._base_executable)) +WINENV = not _path_eq(sys.executable, sys.base_executable) def _close_handles(*handles): @@ -62,7 +61,7 @@ def __init__(self, process_obj): # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): - python_exe = sys._base_executable + python_exe = sys.base_executable env = os.environ.copy() env["__PYVENV_LAUNCHER__"] = sys.executable else: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 4f6c11b2663efd..7c56c68817d9e4 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -25,13 +25,6 @@ except ImportError: ctypes = None -# Platforms that set sys._base_executable can create venvs from within -# another venv, so no need to skip tests that require venv.create(). -requireVenvCreate = unittest.skipUnless( - hasattr(sys, '_base_executable') - or sys.prefix == sys.base_prefix, - 'cannot run venv.create from within a venv on this platform') - def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, @@ -57,7 +50,7 @@ def setUp(self): self.bindir = 'bin' self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' - executable = getattr(sys, '_base_executable', sys.executable) + executable = sys.base_executable self.exe = os.path.split(executable)[-1] def tearDown(self): @@ -102,7 +95,7 @@ def test_defaults(self): else: self.assertFalse(os.path.exists(p)) data = self.get_text_file_contents('pyvenv.cfg') - executable = getattr(sys, '_base_executable', sys.executable) + executable = sys.base_executable path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) fn = self.get_env_file(self.bindir, self.exe) @@ -153,7 +146,6 @@ def pip_cmd_checker(cmd): with patch('venv.subprocess.check_call', pip_cmd_checker): builder.upgrade_dependencies(fake_context) - @requireVenvCreate def test_prefixes(self): """ Test that the prefix values are as expected. @@ -289,7 +281,6 @@ def test_symlinking(self): # run the test, the pyvenv.cfg in the venv created in the test will # point to the venv being used to run the test, and we lose the link # to the source build - so Python can't initialise properly. - @requireVenvCreate def test_executable(self): """ Test that the sys.executable value is as expected. @@ -333,7 +324,6 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') - @requireVenvCreate def test_multiprocessing(self): """ Test that the multiprocessing is able to spawn. @@ -353,7 +343,6 @@ def test_multiprocessing(self): 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) -@requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" def assert_pip_not_installed(self): diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index b64125fa4fe175..a8cf146c82cd72 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -112,7 +112,7 @@ def create_if_needed(d): prompt = self.prompt if self.prompt is not None else context.env_name context.prompt = '(%s) ' % prompt create_if_needed(env_dir) - executable = getattr(sys, '_base_executable', sys.executable) + executable = sys.base_executable dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable context.python_dir = dirname diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 5ae5a123ae5521..5ffe2ebddfd2a5 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -6,6 +6,7 @@ #define WIN32_LEAN_AND_MEAN #include #include +#include #include #include @@ -70,21 +71,23 @@ set_process_name(PyConfig *config) wchar_t *r = NULL; const auto family = get_package_family(); - while (!family.empty() && !r) { - r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); - if (!r) { - Py_FatalError("out of memory"); - return 0; - } - len = _snwprintf_s(r, bufferLen, _TRUNCATE, - L"%ls\\Microsoft\\WindowsApps\\%ls\\%ls", - _wgetenv(L"LOCALAPPDATA"), - family.c_str(), - PROGNAME); - if (len < 0) { - free((void *)r); - r = NULL; - bufferLen *= 2; + + if (!family.empty()) { + PWSTR localAppData; + if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, + NULL, &localAppData))) { + bufferLen = (DWORD)(wcslen(localAppData) + + family.size() + + wcslen(PROGNAME) + + 25); + r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); + swprintf_s(r, bufferLen, + L"%ls\\Microsoft\\WindowsApps\\%ls\\%ls", + localAppData, + family.c_str(), + PROGNAME); + + CoTaskMemFree(localAppData); } } @@ -131,12 +134,12 @@ wmain(int argc, wchar_t **argv) if (PyStatus_Exception(status)) { goto fail; } - + status = PyConfig_InitPythonConfig(&config); if (PyStatus_Exception(status)) { goto fail; } - + status = PyConfig_SetArgv(&config, argc, argv); if (PyStatus_Exception(status)) { goto fail; From a9163cae68154efd8e887e7ef216adf064d3ffa3 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 10:08:30 -0700 Subject: [PATCH 03/16] Switch to std::wstring because I'm sick of memory management --- Lib/venv/__init__.py | 2 +- PC/python_uwp.cpp | 104 ++++++++++++++++++++++++------------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index a8cf146c82cd72..e4ad06013ec74b 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -251,7 +251,7 @@ def setup_python(self, context): for suffix in suffixes: src = os.path.join(dirname, suffix) - if os.path.exists(src): + if os.path.lexists(src): copier(src, os.path.join(binpath, suffix)) if sysconfig.is_python_build(True): diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 5ffe2ebddfd2a5..aead9943d0df68 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -8,6 +8,8 @@ #include #include +#include + #include #include @@ -66,9 +68,7 @@ get_package_family() static int set_process_name(PyConfig *config) { - DWORD bufferLen = MAX_PATH; - DWORD len = bufferLen; - wchar_t *r = NULL; + std::wstring executable, home; const auto family = get_package_family(); @@ -76,47 +76,55 @@ set_process_name(PyConfig *config) PWSTR localAppData; if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, NULL, &localAppData))) { - bufferLen = (DWORD)(wcslen(localAppData) - + family.size() - + wcslen(PROGNAME) - + 25); - r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); - swprintf_s(r, bufferLen, - L"%ls\\Microsoft\\WindowsApps\\%ls\\%ls", - localAppData, - family.c_str(), - PROGNAME); + executable = std::wstring(localAppData) + + L"\\Microsoft\\WindowsApps\\" + + family + + L"\\" + + PROGNAME; CoTaskMemFree(localAppData); } } - while (!r) { - r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); - if (!r) { - Py_FatalError("out of memory"); - return NULL; - } - len = GetModuleFileNameW(NULL, r, bufferLen); + home.resize(MAX_PATH); + while (true) { + DWORD len = GetModuleFileNameW( + NULL, home.data(), (DWORD)home.size()); if (len == 0) { - free((void *)r); - return 0; - } else if (len == bufferLen && + home.clear(); + break; + } else if (len == home.size() && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - free(r); - r = NULL; - bufferLen *= 2; + home.resize(len * 2); + } else { + home.resize(len); + size_t bslash = home.find_last_of(L"/\\"); + if (bslash != std::wstring::npos) { + home.erase(bslash); + } + break; } } - PyConfig_SetString(config, &config->base_executable, r); - const wchar_t *launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); - if (launcherPath) { - PyConfig_SetString(config, &config->executable, launcherPath); - } else { - PyConfig_SetString(config, &config->executable, r); + if (executable.empty() && !home.empty()) { + executable = home + L"\\" + PROGNAME; + } + + if (!home.empty()) { + PyConfig_SetString(config, &config->home, home.c_str()); + // XXX: Why doesn't setting prefix and exec_prefix work? + //PyConfig_SetString(config, &config->prefix, home.c_str()); + //PyConfig_SetString(config, &config->exec_prefix, home.c_str()); + } + if (!executable.empty()) { + PyConfig_SetString(config, &config->base_executable, executable.c_str()); + const wchar_t *launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); + if (launcherPath) { + PyConfig_SetString(config, &config->executable, launcherPath); + } else { + PyConfig_SetString(config, &config->executable, executable.c_str()); + } } - free((void *)r); return 1; } @@ -156,21 +164,23 @@ wmain(int argc, wchar_t **argv) p = argv[0]; } if (p) { - if (*p++ == L'\\') { - const wchar_t *moduleName = NULL; - if (wcsnicmp(p, L"pip", 3) == 0) { - moduleName = L"pip"; - /* No longer required when pip 19.1 is added */ - _wputenv_s(L"PIP_USER", L"true"); - } else if (wcsnicmp(p, L"idle", 4) == 0) { - moduleName = L"idlelib"; - } + if (*p == L'\\') { + p++; + } - if (moduleName) { - PyConfig_SetString(&config, &config.run_module, moduleName); - PyConfig_SetString(&config, &config.run_filename, NULL); - PyConfig_SetString(&config, &config.run_command, NULL); - } + const wchar_t *moduleName = NULL; + if (wcsnicmp(p, L"pip", 3) == 0) { + moduleName = L"pip"; + /* No longer required when pip 19.1 is added */ + _wputenv_s(L"PIP_USER", L"true"); + } else if (wcsnicmp(p, L"idle", 4) == 0) { + moduleName = L"idlelib"; + } + + if (moduleName) { + PyConfig_SetString(&config, &config.run_module, moduleName); + PyConfig_SetString(&config, &config.run_filename, NULL); + PyConfig_SetString(&config, &config.run_command, NULL); } } From 5bf292be3abce9b89e754b49a12acd387f06d33e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 10:54:24 -0700 Subject: [PATCH 04/16] Fix config.base_executable calculations --- Include/internal/pycore_pathconfig.h | 2 ++ Lib/site.py | 13 ++++++------- Lib/test/test_embed.py | 17 ++++++++++------- Modules/getpath.c | 12 ++++++++++++ PC/getpathp.c | 10 ++++++++++ Python/pathconfig.c | 15 +++++++++++++++ 6 files changed, 55 insertions(+), 14 deletions(-) diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index be12c6f5cf321a..a69868e2cdac01 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -27,6 +27,8 @@ typedef struct _PyPathConfig { are ignored when their value are equal to -1 (unset). */ int isolated; int site_import; + /* sys.base_executable if in a venv */ + wchar_t *base_executable; } _PyPathConfig; #define _PyPathConfig_INIT \ diff --git a/Lib/site.py b/Lib/site.py index e7aafb7011cfc7..0105a57c4ef985 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -458,14 +458,13 @@ def venv(known_paths): env = os.environ if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: - executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__'] - elif sys.platform == 'win32' and '__PYVENV_LAUNCHER__' in env: + executable = sys.base_executable = os.environ['__PYVENV_LAUNCHER__'] + elif sys.platform == 'win32': executable = sys.executable - import _winapi - sys._base_executable = _winapi.GetModuleFileName(0) - # bpo-35873: Clear the environment variable to avoid it being - # inherited by child processes. - del os.environ['__PYVENV_LAUNCHER__'] + if '__PYVENV_LAUNCHER__' in env: + # bpo-35873: Clear the environment variable to avoid it being + # inherited by child processes. + del os.environ['__PYVENV_LAUNCHER__'] else: executable = sys.executable exe_dir, _ = os.path.split(os.path.abspath(executable)) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index b89748938ba772..d5eefc0558333f 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -362,6 +362,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'pythonpath_env': None, 'home': None, 'executable': GET_DEFAULT_CONFIG, + 'base_executable': GET_DEFAULT_CONFIG, 'prefix': GET_DEFAULT_CONFIG, 'base_prefix': GET_DEFAULT_CONFIG, @@ -534,14 +535,16 @@ def get_expected_config(self, expected_preconfig, expected, env, api, if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG: expected['stdio_errors'] = 'surrogateescape' + if sys.platform == 'win32': + default_executable = self.test_exe + elif expected['program_name'] is not self.GET_DEFAULT_CONFIG: + default_executable = os.path.abspath(expected['program_name']) + else: + default_executable = os.path.join(os.getcwd(), '_testembed') if expected['executable'] is self.GET_DEFAULT_CONFIG: - if sys.platform == 'win32': - expected['executable'] = self.test_exe - else: - if expected['program_name'] is not self.GET_DEFAULT_CONFIG: - expected['executable'] = os.path.abspath(expected['program_name']) - else: - expected['executable'] = os.path.join(os.getcwd(), '_testembed') + expected['executable'] = default_executable + if expected['base_executable'] is self.GET_DEFAULT_CONFIG: + expected['base_executable'] = default_executable if expected['program_name'] is self.GET_DEFAULT_CONFIG: expected['program_name'] = './_testembed' diff --git a/Modules/getpath.c b/Modules/getpath.c index 751c0b79e8fafa..710e57e118c820 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -129,10 +129,12 @@ typedef struct { wchar_t *lib_python; /* "lib/pythonX.Y" */ wchar_t argv0_path[MAXPATHLEN+1]; + wchar_t base_executable[MAXPATHLEN+1]; wchar_t zip_path[MAXPATHLEN+1]; /* ".../lib/pythonXY.zip" */ int prefix_found; /* found platform independent libraries? */ int exec_prefix_found; /* found the platform dependent libraries? */ + int base_executable_found; /* found a base_executable path */ } PyCalculatePath; static const wchar_t delimiter[2] = {DELIM, '\0'}; @@ -987,6 +989,12 @@ calculate_read_pyenv(PyCalculatePath *calculate) /* Look for a 'home' variable and set argv0_path to it, if found */ if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) { + /* preserve the previous argv0_path as sys.base_executable */ + if (safe_wcscpy(calculate->base_executable, calculate->argv0_path, + Py_ARRAY_LENGTH(calculate->base_executable)) < 0) { + return PATHLEN_ERR(); + } + calculate->base_executable_found = 1; if (safe_wcscpy(calculate->argv0_path, tmpbuffer, Py_ARRAY_LENGTH(calculate->argv0_path)) < 0) { return PATHLEN_ERR(); @@ -1240,6 +1248,10 @@ calculate_path_impl(const PyConfig *config, return _PyStatus_NO_MEMORY(); } + if (calculate->base_executable_found) { + pathconfig->base_executable = _PyMem_RawWcsdup(calculate->base_executable); + } + return _PyStatus_OK(); } diff --git a/PC/getpathp.c b/PC/getpathp.c index e86cf13a4910ac..4a0aee0392b427 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -128,6 +128,7 @@ typedef struct { wchar_t argv0_path[MAXPATHLEN+1]; wchar_t zip_path[MAXPATHLEN+1]; + wchar_t base_executable[MAXPATHLEN+1]; } PyCalculatePath; @@ -729,6 +730,8 @@ calculate_pyvenv_file(PyCalculatePath *calculate) /* Look for a 'home' variable and set argv0_path to it, if found */ wchar_t tmpbuffer[MAXPATHLEN+1]; if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) { + /* Preserve original path as base_executable */ + wcscpy_s(calculate->base_executable, MAXPATHLEN+1, calculate->argv0_path); wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer); } fclose(env_file); @@ -999,6 +1002,13 @@ calculate_path_impl(const PyConfig *config, if (pathconfig->exec_prefix == NULL) { return _PyStatus_NO_MEMORY(); } + if (calculate->base_executable[0]) { + pathconfig->base_executable = _PyMem_RawWcsdup( + calculate->base_executable); + if (pathconfig->base_executable == NULL) { + return _PyStatus_NO_MEMORY(); + } + } return _PyStatus_OK(); } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index fea397cca22090..3333ce771c0107 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -57,6 +57,7 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->module_search_path); CLEAR(config->home); CLEAR(config->program_name); + CLEAR(config->base_executable); #undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -89,6 +90,10 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config) status = _PyStatus_NO_MEMORY(); goto error; } + if (copy_wstr(&new_config.base_executable, config->base_executable) < 0) { + status = _PyStatus_NO_MEMORY(); + goto error; + } pathconfig_clear(pathconfig); *pathconfig = new_config; @@ -224,6 +229,9 @@ _PyConfig_SetPathConfig(const PyConfig *config) if (copy_wstr(&pathconfig.home, config->home) < 0) { goto no_memory; } + if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) { + goto no_memory; + } status = _PyPathConfig_SetGlobal(&pathconfig); if (_PyStatus_EXCEPTION(status)) { @@ -321,6 +329,13 @@ config_calculate_pathconfig(PyConfig *config) } } + if (config->base_executable == NULL) { + if (copy_wstr(&config->base_executable, + pathconfig.base_executable) < 0) { + goto no_memory; + } + } + if (pathconfig.isolated != -1) { config->isolated = pathconfig.isolated; } From adea10bf084b6ac1a7fe06c56b4284bb931dd096 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 11:03:46 -0700 Subject: [PATCH 05/16] Fix trailing whitespace --- Lib/test/test_embed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index d5eefc0558333f..9c78aa059fc34d 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -544,7 +544,7 @@ def get_expected_config(self, expected_preconfig, expected, env, api, if expected['executable'] is self.GET_DEFAULT_CONFIG: expected['executable'] = default_executable if expected['base_executable'] is self.GET_DEFAULT_CONFIG: - expected['base_executable'] = default_executable + expected['base_executable'] = default_executable if expected['program_name'] is self.GET_DEFAULT_CONFIG: expected['program_name'] = './_testembed' From 75ebe6ad95c086a17f1e2b75f7dbb99bb4ffa22b Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 11:36:16 -0700 Subject: [PATCH 06/16] Revert base_executable changes in path config --- Include/internal/pycore_pathconfig.h | 2 -- Modules/getpath.c | 12 ------------ PC/getpathp.c | 10 ---------- Python/pathconfig.c | 21 --------------------- 4 files changed, 45 deletions(-) diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index a69868e2cdac01..be12c6f5cf321a 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -27,8 +27,6 @@ typedef struct _PyPathConfig { are ignored when their value are equal to -1 (unset). */ int isolated; int site_import; - /* sys.base_executable if in a venv */ - wchar_t *base_executable; } _PyPathConfig; #define _PyPathConfig_INIT \ diff --git a/Modules/getpath.c b/Modules/getpath.c index 710e57e118c820..751c0b79e8fafa 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -129,12 +129,10 @@ typedef struct { wchar_t *lib_python; /* "lib/pythonX.Y" */ wchar_t argv0_path[MAXPATHLEN+1]; - wchar_t base_executable[MAXPATHLEN+1]; wchar_t zip_path[MAXPATHLEN+1]; /* ".../lib/pythonXY.zip" */ int prefix_found; /* found platform independent libraries? */ int exec_prefix_found; /* found the platform dependent libraries? */ - int base_executable_found; /* found a base_executable path */ } PyCalculatePath; static const wchar_t delimiter[2] = {DELIM, '\0'}; @@ -989,12 +987,6 @@ calculate_read_pyenv(PyCalculatePath *calculate) /* Look for a 'home' variable and set argv0_path to it, if found */ if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, buflen)) { - /* preserve the previous argv0_path as sys.base_executable */ - if (safe_wcscpy(calculate->base_executable, calculate->argv0_path, - Py_ARRAY_LENGTH(calculate->base_executable)) < 0) { - return PATHLEN_ERR(); - } - calculate->base_executable_found = 1; if (safe_wcscpy(calculate->argv0_path, tmpbuffer, Py_ARRAY_LENGTH(calculate->argv0_path)) < 0) { return PATHLEN_ERR(); @@ -1248,10 +1240,6 @@ calculate_path_impl(const PyConfig *config, return _PyStatus_NO_MEMORY(); } - if (calculate->base_executable_found) { - pathconfig->base_executable = _PyMem_RawWcsdup(calculate->base_executable); - } - return _PyStatus_OK(); } diff --git a/PC/getpathp.c b/PC/getpathp.c index 4a0aee0392b427..e86cf13a4910ac 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -128,7 +128,6 @@ typedef struct { wchar_t argv0_path[MAXPATHLEN+1]; wchar_t zip_path[MAXPATHLEN+1]; - wchar_t base_executable[MAXPATHLEN+1]; } PyCalculatePath; @@ -730,8 +729,6 @@ calculate_pyvenv_file(PyCalculatePath *calculate) /* Look for a 'home' variable and set argv0_path to it, if found */ wchar_t tmpbuffer[MAXPATHLEN+1]; if (_Py_FindEnvConfigValue(env_file, L"home", tmpbuffer, MAXPATHLEN)) { - /* Preserve original path as base_executable */ - wcscpy_s(calculate->base_executable, MAXPATHLEN+1, calculate->argv0_path); wcscpy_s(calculate->argv0_path, MAXPATHLEN+1, tmpbuffer); } fclose(env_file); @@ -1002,13 +999,6 @@ calculate_path_impl(const PyConfig *config, if (pathconfig->exec_prefix == NULL) { return _PyStatus_NO_MEMORY(); } - if (calculate->base_executable[0]) { - pathconfig->base_executable = _PyMem_RawWcsdup( - calculate->base_executable); - if (pathconfig->base_executable == NULL) { - return _PyStatus_NO_MEMORY(); - } - } return _PyStatus_OK(); } diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 3333ce771c0107..ec67405a28d054 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -57,7 +57,6 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->module_search_path); CLEAR(config->home); CLEAR(config->program_name); - CLEAR(config->base_executable); #undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -90,10 +89,6 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config) status = _PyStatus_NO_MEMORY(); goto error; } - if (copy_wstr(&new_config.base_executable, config->base_executable) < 0) { - status = _PyStatus_NO_MEMORY(); - goto error; - } pathconfig_clear(pathconfig); *pathconfig = new_config; @@ -229,9 +224,6 @@ _PyConfig_SetPathConfig(const PyConfig *config) if (copy_wstr(&pathconfig.home, config->home) < 0) { goto no_memory; } - if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) { - goto no_memory; - } status = _PyPathConfig_SetGlobal(&pathconfig); if (_PyStatus_EXCEPTION(status)) { @@ -329,13 +321,6 @@ config_calculate_pathconfig(PyConfig *config) } } - if (config->base_executable == NULL) { - if (copy_wstr(&config->base_executable, - pathconfig.base_executable) < 0) { - goto no_memory; - } - } - if (pathconfig.isolated != -1) { config->isolated = pathconfig.isolated; } @@ -370,12 +355,6 @@ _PyConfig_InitPathConfig(PyConfig *config) } } - if (config->base_executable == NULL) { - if (copy_wstr(&config->base_executable, config->executable) < 0) { - return _PyStatus_NO_MEMORY(); - } - } - if (config->base_prefix == NULL) { if (copy_wstr(&config->base_prefix, config->prefix) < 0) { return _PyStatus_NO_MEMORY(); From 00c5e83fec2dad43a6a5676961a6b55512bb514e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 11:40:26 -0700 Subject: [PATCH 07/16] Restore one that I should have left --- Python/pathconfig.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Python/pathconfig.c b/Python/pathconfig.c index ec67405a28d054..92d083a970f71c 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -367,6 +367,14 @@ _PyConfig_InitPathConfig(PyConfig *config) return _PyStatus_NO_MEMORY(); } } + + if (config->base_executable == NULL) { + if (copy_wstr(&config->base_executable, + config->executable) < 0) { + return _PyStatus_NO_MEMORY(); + } + } + return _PyStatus_OK(); } From 9a744e805ea1a9d9ee9d45c04dfa36e92884e29e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 12:21:40 -0700 Subject: [PATCH 08/16] Make sys._base_executable internal again and calculate it properly on Windows --- Include/internal/pycore_pathconfig.h | 2 ++ Lib/multiprocessing/popen_spawn_win32.py | 5 +++-- Lib/site.py | 2 +- Lib/test/test_venv.py | 13 ++++++++++++- Lib/venv/__init__.py | 2 +- PC/getpathp.c | 17 ++++++++++++++--- Python/pathconfig.c | 22 ++++++++++++++++++++++ Python/sysmodule.c | 2 +- 8 files changed, 56 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index be12c6f5cf321a..9e0ba0b01a805f 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -27,6 +27,8 @@ typedef struct _PyPathConfig { are ignored when their value are equal to -1 (unset). */ int isolated; int site_import; + /* Set when a venv is detected */ + wchar_t *base_executable; } _PyPathConfig; #define _PyPathConfig_INIT \ diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 136b8b3a96e96a..de4c5ecf1fa083 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -22,7 +22,8 @@ def _path_eq(p1, p2): return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2) -WINENV = not _path_eq(sys.executable, sys.base_executable) +WINENV = (hasattr(sys, '_base_executable') and + not _path_eq(sys.executable, sys._base_executable)) def _close_handles(*handles): @@ -61,7 +62,7 @@ def __init__(self, process_obj): # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): - python_exe = sys.base_executable + python_exe = sys._base_executable env = os.environ.copy() env["__PYVENV_LAUNCHER__"] = sys.executable else: diff --git a/Lib/site.py b/Lib/site.py index 0105a57c4ef985..f574906efffac8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -458,7 +458,7 @@ def venv(known_paths): env = os.environ if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: - executable = sys.base_executable = os.environ['__PYVENV_LAUNCHER__'] + executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__'] elif sys.platform == 'win32': executable = sys.executable if '__PYVENV_LAUNCHER__' in env: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 7c56c68817d9e4..d57781554f02d0 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -25,6 +25,13 @@ except ImportError: ctypes = None +# Platforms that set sys._base_executable can create venvs from within +# another venv, so no need to skip tests that require venv.create(). +requireVenvCreate = unittest.skipUnless( + hasattr(sys, '_base_executable') + or sys.prefix == sys.base_prefix, + 'cannot run venv.create from within a venv on this platform') + def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, @@ -50,7 +57,7 @@ def setUp(self): self.bindir = 'bin' self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' - executable = sys.base_executable + executable = getattr(sys, '_base_executable', sys.executable) self.exe = os.path.split(executable)[-1] def tearDown(self): @@ -146,6 +153,7 @@ def pip_cmd_checker(cmd): with patch('venv.subprocess.check_call', pip_cmd_checker): builder.upgrade_dependencies(fake_context) + @requireVenvCreate def test_prefixes(self): """ Test that the prefix values are as expected. @@ -281,6 +289,7 @@ def test_symlinking(self): # run the test, the pyvenv.cfg in the venv created in the test will # point to the venv being used to run the test, and we lose the link # to the source build - so Python can't initialise properly. + @requireVenvCreate def test_executable(self): """ Test that the sys.executable value is as expected. @@ -324,6 +333,7 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + @requireVenvCreate def test_multiprocessing(self): """ Test that the multiprocessing is able to spawn. @@ -343,6 +353,7 @@ def test_multiprocessing(self): 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) +@requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" def assert_pip_not_installed(self): diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index e4ad06013ec74b..778efaa9b565fc 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -112,7 +112,7 @@ def create_if_needed(d): prompt = self.prompt if self.prompt is not None else context.env_name context.prompt = '(%s) ' % prompt create_if_needed(env_dir) - executable = sys.base_executable + executable = getattr(sys, '_base_executable', sys.executable) dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable context.python_dir = dirname diff --git a/PC/getpathp.c b/PC/getpathp.c index e86cf13a4910ac..fb1e0a1d80f379 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -537,14 +537,25 @@ get_program_full_path(const PyConfig *config, wchar_t program_full_path[MAXPATHLEN+1]; memset(program_full_path, 0, sizeof(program_full_path)); + if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { + /* GetModuleFileName should never fail when passed NULL */ + return _PyStatus_ERR("Cannot determine program path"); + } + /* The launcher may need to force the executable path to a * different environment, so override it here. */ pyvenv_launcher = _wgetenv(L"__PYVENV_LAUNCHER__"); if (pyvenv_launcher && pyvenv_launcher[0]) { + /* If overridden, preserve the original full path */ + pathconfig->base_executable = PyMem_RawMalloc( + sizeof(wchar_t) * (MAXPATHLEN + 1)); + + PyStatus status = canonicalize(pathconfig->base_executable, + program_full_path); + if (_PyStatus_EXCEPTION(status)) { + return status; + } wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher); - } else if (!GetModuleFileNameW(NULL, program_full_path, MAXPATHLEN)) { - /* GetModuleFileName should never fail when passed NULL */ - return _PyStatus_ERR("Cannot determine program path"); } pathconfig->program_full_path = PyMem_RawMalloc( diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 92d083a970f71c..79ec4af00d83ba 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -57,6 +57,7 @@ pathconfig_clear(_PyPathConfig *config) CLEAR(config->module_search_path); CLEAR(config->home); CLEAR(config->program_name); + CLEAR(config->base_executable); #undef CLEAR PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -89,6 +90,14 @@ pathconfig_calculate(_PyPathConfig *pathconfig, const PyConfig *config) status = _PyStatus_NO_MEMORY(); goto error; } + if (config->base_executable) { + PyMem_RawFree(new_config.base_executable); + if (copy_wstr(&new_config.base_executable, + config->base_executable) < 0) { + status = _PyStatus_NO_MEMORY(); + goto error; + } + } pathconfig_clear(pathconfig); *pathconfig = new_config; @@ -132,6 +141,7 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) COPY_ATTR(module_search_path); COPY_ATTR(program_name); COPY_ATTR(home); + COPY_ATTR(base_executable); pathconfig_clear(&_Py_path_config); /* Steal new_config strings; don't clear new_config */ @@ -224,6 +234,9 @@ _PyConfig_SetPathConfig(const PyConfig *config) if (copy_wstr(&pathconfig.home, config->home) < 0) { goto no_memory; } + if (copy_wstr(&pathconfig.base_executable, config->base_executable) < 0) { + goto no_memory; + } status = _PyPathConfig_SetGlobal(&pathconfig); if (_PyStatus_EXCEPTION(status)) { @@ -321,6 +334,13 @@ config_calculate_pathconfig(PyConfig *config) } } + if (config->base_executable == NULL) { + if (copy_wstr(&config->base_executable, + pathconfig.base_executable) < 0) { + goto no_memory; + } + } + if (pathconfig.isolated != -1) { config->isolated = pathconfig.isolated; } @@ -442,6 +462,8 @@ Py_SetPath(const wchar_t *path) _Py_path_config.home = NULL; new_config.program_name = _Py_path_config.program_name; _Py_path_config.program_name = NULL; + new_config.base_executable = _Py_path_config.base_executable; + _Py_path_config.base_executable = NULL; pathconfig_clear(&_Py_path_config); _Py_path_config = new_config; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index aa5834aac366d1..c919492e326209 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2850,7 +2850,7 @@ _PySys_InitMain(_PyRuntimeState *runtime, PyThreadState *tstate) COPY_LIST("path", config->module_search_paths); SET_SYS_FROM_WSTR("executable", config->executable); - SET_SYS_FROM_WSTR("base_executable", config->base_executable); + SET_SYS_FROM_WSTR("_base_executable", config->base_executable); SET_SYS_FROM_WSTR("prefix", config->prefix); SET_SYS_FROM_WSTR("base_prefix", config->base_prefix); SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix); From b68dc7d2bacd3226050f55087e86a58216b6a04f Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 27 Jun 2019 12:27:21 -0700 Subject: [PATCH 09/16] Fix venv test --- Lib/test/test_venv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index d57781554f02d0..4f6c11b2663efd 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -102,7 +102,7 @@ def test_defaults(self): else: self.assertFalse(os.path.exists(p)) data = self.get_text_file_contents('pyvenv.cfg') - executable = sys.base_executable + executable = getattr(sys, '_base_executable', sys.executable) path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) fn = self.get_env_file(self.bindir, self.exe) From d5fcb036992374899871c70d8b7317921b8cb85e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 28 Jun 2019 10:33:46 -0700 Subject: [PATCH 10/16] Address feedback and fix venv with symlinks --- Include/cpython/initconfig.h | 2 +- Lib/site.py | 6 -- Lib/test/test_venv.py | 10 +-- Lib/venv/__init__.py | 89 +++++++++++-------- .../2019-06-28-09-44-08.bpo-37369.1iVpxq.rst | 1 + PC/getpathp.c | 5 +- PC/python_uwp.cpp | 78 ++++++++++------ 7 files changed, 114 insertions(+), 77 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2019-06-28-09-44-08.bpo-37369.1iVpxq.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 5fb6fade553731..297fbf70792f31 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -374,7 +374,7 @@ typedef struct { to zero. */ wchar_t *executable; /* sys.executable */ - wchar_t *base_executable; /* sys.base_executable */ + wchar_t *base_executable; /* sys._base_executable */ wchar_t *prefix; /* sys.prefix */ wchar_t *base_prefix; /* sys.base_prefix */ wchar_t *exec_prefix; /* sys.exec_prefix */ diff --git a/Lib/site.py b/Lib/site.py index f574906efffac8..a065ab0b5db58f 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -459,12 +459,6 @@ def venv(known_paths): env = os.environ if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__'] - elif sys.platform == 'win32': - executable = sys.executable - if '__PYVENV_LAUNCHER__' in env: - # bpo-35873: Clear the environment variable to avoid it being - # inherited by child processes. - del os.environ['__PYVENV_LAUNCHER__'] else: executable = sys.executable exe_dir, _ = os.path.split(os.path.abspath(executable)) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 4f6c11b2663efd..e1a4d134e2f4e2 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -158,10 +158,6 @@ def test_prefixes(self): """ Test that the prefix values are as expected. """ - #check our prefixes - self.assertEqual(sys.base_prefix, sys.prefix) - self.assertEqual(sys.base_exec_prefix, sys.exec_prefix) - # check a venv's prefixes rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) @@ -169,9 +165,9 @@ def test_prefixes(self): cmd = [envpy, '-c', None] for prefix, expected in ( ('prefix', self.env_dir), - ('prefix', self.env_dir), - ('base_prefix', sys.prefix), - ('base_exec_prefix', sys.exec_prefix)): + ('exec_prefix', self.env_dir), + ('base_prefix', sys.base_prefix), + ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) self.assertEqual(out.strip(), expected.encode()) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 778efaa9b565fc..2189d6ac4f9894 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -163,47 +163,66 @@ def create_configuration(self, context): if self.prompt is not None: f.write(f'prompt = {self.prompt!r}\n') - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! + if os.name != 'nt': + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + """ + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! + if relative_symlinks_ok: + assert os.path.dirname(src) == os.path.dirname(dst) + os.symlink(os.path.basename(src), dst) + else: + os.symlink(src, dst) + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: + shutil.copyfile(src, dst) + else: + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + """ + bad_src = os.path.lexists(src) and not os.path.exists(src) + if self.symlinks and not bad_src and not os.path.islink(dst): + try: if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - if os.name == 'nt': - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - logger.warning('Unable to copy %r', src) return + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + + # On Windows, we rewrite symlinks to our base python.exe into + # copies of venvlauncher.exe + basename, ext = os.path.splitext(os.path.basename(src)) + srcfn = os.path.join(os.path.dirname(__file__), + "scripts", + "nt", + basename + ext) + # Builds or venv's from builds need to remap source file + # locations, as we do not put them into Lib/venv/scripts + if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): + if basename.endswith('_d'): + ext = '_d' + ext + basename = basename[:-2] + if basename == 'python': + basename = 'venvlauncher' + elif basename == 'pythonw': + basename = 'venvwlauncher' + src = os.path.join(os.path.dirname(src), basename + ext) + else: + src = srcfn + if not os.path.exists(src): + if not bad_src: + logger.warning('Unable to copy %r', src) + return shutil.copyfile(src, dst) diff --git a/Misc/NEWS.d/next/Windows/2019-06-28-09-44-08.bpo-37369.1iVpxq.rst b/Misc/NEWS.d/next/Windows/2019-06-28-09-44-08.bpo-37369.1iVpxq.rst new file mode 100644 index 00000000000000..5eaed61a9261b9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2019-06-28-09-44-08.bpo-37369.1iVpxq.rst @@ -0,0 +1 @@ +Fixes path for :data:`sys.executable` when running from the Microsoft Store. diff --git a/PC/getpathp.c b/PC/getpathp.c index fb1e0a1d80f379..7465d314b04398 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -549,13 +549,16 @@ get_program_full_path(const PyConfig *config, /* If overridden, preserve the original full path */ pathconfig->base_executable = PyMem_RawMalloc( sizeof(wchar_t) * (MAXPATHLEN + 1)); - PyStatus status = canonicalize(pathconfig->base_executable, program_full_path); if (_PyStatus_EXCEPTION(status)) { return status; } + wcscpy_s(program_full_path, MAXPATHLEN+1, pyvenv_launcher); + /* bpo-35873: Clear the environment variable to avoid it being + * inherited by child processes. */ + _wputenv_s(L"__PYVENV_LAUNCHER__", L""); } pathconfig->program_full_path = PyMem_RawMalloc( diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index aead9943d0df68..2dfd2256ed6dd0 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -27,51 +27,51 @@ const wchar_t *PROGNAME = L"python.exe"; #endif #endif -static void -set_user_base() +static std::wstring +get_user_base() { - wchar_t envBuffer[2048]; try { const auto appData = winrt::Windows::Storage::ApplicationData::Current(); if (appData) { const auto localCache = appData.LocalCacheFolder(); if (localCache) { auto path = localCache.Path(); - if (!path.empty() && - !wcscpy_s(envBuffer, path.c_str()) && - !wcscat_s(envBuffer, L"\\local-packages") - ) { - _wputenv_s(L"PYTHONUSERBASE", envBuffer); + if (!path.empty()) { + return std::wstring(path) + L"\\local-packages"; } } } } catch (...) { } + return std::wstring(); } -static winrt::hstring +static std::wstring get_package_family() { try { const auto package = winrt::Windows::ApplicationModel::Package::Current(); if (package) { const auto id = package.Id(); - return id ? id.FamilyName() : winrt::hstring(); + if (id) { + return std::wstring(id.FamilyName()); + } } } catch (...) { } - return winrt::hstring(); + return std::wstring(); } -static int +static PyStatus set_process_name(PyConfig *config) { + PyStatus status; std::wstring executable, home; const auto family = get_package_family(); - + if (!family.empty()) { PWSTR localAppData; if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, @@ -101,6 +101,8 @@ set_process_name(PyConfig *config) size_t bslash = home.find_last_of(L"/\\"); if (bslash != std::wstring::npos) { home.erase(bslash); + } else { + home.clear(); } break; } @@ -111,22 +113,34 @@ set_process_name(PyConfig *config) } if (!home.empty()) { - PyConfig_SetString(config, &config->home, home.c_str()); - // XXX: Why doesn't setting prefix and exec_prefix work? - //PyConfig_SetString(config, &config->prefix, home.c_str()); - //PyConfig_SetString(config, &config->exec_prefix, home.c_str()); + status = PyConfig_SetString(config, &config->home, home.c_str()); + if (PyStatus_Exception(status)) { + return status; + } } if (!executable.empty()) { - PyConfig_SetString(config, &config->base_executable, executable.c_str()); + status = PyConfig_SetString(config, &config->base_executable, executable.c_str()); + if (PyStatus_Exception(status)) { + return status; + } + const wchar_t *launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); if (launcherPath) { - PyConfig_SetString(config, &config->executable, launcherPath); + status = PyConfig_SetString( + config, &config->executable, launcherPath); + /* bpo-35873: Clear the environment variable to avoid it being + * inherited by child processes. */ + _wputenv_s(L"__PYVENV_LAUNCHER__", L""); } else { - PyConfig_SetString(config, &config->executable, executable.c_str()); + status = PyConfig_SetString( + config, &config->executable, executable.c_str()); + } + if (PyStatus_Exception(status)) { + return status; } } - return 1; + return PyStatus_Ok(); } int @@ -153,11 +167,12 @@ wmain(int argc, wchar_t **argv) goto fail; } - if (!set_process_name(&config)) { - status = PyStatus_Exit(121); + status = set_process_name(&config); + if (PyStatus_Exception(status)) { goto fail; } - set_user_base(); + + _wputenv_s(L"PYTHONUSERBASE", get_user_base().c_str()); const wchar_t *p = wcsrchr(argv[0], L'\\'); if (!p) { @@ -178,9 +193,18 @@ wmain(int argc, wchar_t **argv) } if (moduleName) { - PyConfig_SetString(&config, &config.run_module, moduleName); - PyConfig_SetString(&config, &config.run_filename, NULL); - PyConfig_SetString(&config, &config.run_command, NULL); + status = PyConfig_SetString(&config, &config.run_module, moduleName); + if (PyStatus_Exception(status)) { + goto fail; + } + status = PyConfig_SetString(&config, &config.run_filename, NULL); + if (PyStatus_Exception(status)) { + goto fail; + } + status = PyConfig_SetString(&config, &config.run_command, NULL); + if (PyStatus_Exception(status)) { + goto fail; + } } } From 5891ad0e4dba6e372f951a3b4d4bb4d36e17e709 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 28 Jun 2019 11:14:13 -0700 Subject: [PATCH 11/16] Fixes tests that use symlinks --- Lib/test/test_platform.py | 24 ++++++++++++++++++------ Lib/test/test_sysconfig.py | 18 +++++++++++++----- Lib/test/test_venv.py | 7 ++++++- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 9cf17726d92e0d..7c8133a9b7a968 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -20,14 +20,24 @@ def test_architecture(self): @support.skip_unless_symlink def test_architecture_via_symlink(self): # issue3762 + if sys.platform == "win32" and not os.path.exists(sys.executable): + # App symlink appears to not exist, but we want the + # real executable here anyway + import _winapi + real = _winapi.GetModuleFileName(0) + else: + real = os.path.realpath(sys.executable) + link = os.path.abspath(support.TESTFN) + os.symlink(real, link) + # On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at # so we add the directory to the path, PYTHONHOME and PYTHONPATH. env = None if sys.platform == "win32": env = {k.upper(): os.environ[k] for k in os.environ} env["PATH"] = "{};{}".format( - os.path.dirname(sys.executable), env.get("PATH", "")) - env["PYTHONHOME"] = os.path.dirname(sys.executable) + os.path.dirname(real), env.get("PATH", "")) + env["PYTHONHOME"] = os.path.dirname(real) if sysconfig.is_python_build(True): env["PYTHONPATH"] = os.path.dirname(os.__file__) @@ -44,11 +54,8 @@ def get(python, env=None): .format(p.returncode)) return r - real = os.path.realpath(sys.executable) - link = os.path.abspath(support.TESTFN) - os.symlink(real, link) try: - self.assertEqual(get(real), get(link, env=env)) + self.assertEqual(get(sys.executable), get(link, env=env)) finally: os.remove(link) @@ -275,6 +282,11 @@ def test_libc_ver(self): os.path.exists(sys.executable+'.exe'): # Cygwin horror executable = sys.executable + '.exe' + elif sys.platform == "win32" and not os.path.exists(sys.executable): + # App symlink appears to not exist, but we want the + # real executable here anyway + import _winapi + executable = _winapi.GetModuleFileName(0) else: executable = sys.executable platform.libc_ver(executable) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 1b1929885edd6b..d612cdef95598f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -233,16 +233,26 @@ def test_get_scheme_names(self): @skip_unless_symlink def test_symlink(self): + if sys.platform == "win32" and not os.path.exists(sys.executable): + # App symlink appears to not exist, but we want the + # real executable here anyway + import _winapi + real = _winapi.GetModuleFileName(0) + else: + real = os.path.realpath(sys.executable) + link = os.path.abspath(TESTFN) + os.symlink(real, link) + # On Windows, the EXE needs to know where pythonXY.dll is at so we have # to add the directory to the path. env = None if sys.platform == "win32": env = {k.upper(): os.environ[k] for k in os.environ} env["PATH"] = "{};{}".format( - os.path.dirname(sys.executable), env.get("PATH", "")) + os.path.dirname(real), env.get("PATH", "")) # Requires PYTHONHOME as well since we locate stdlib from the # EXE path and not the DLL path (which should be fixed) - env["PYTHONHOME"] = os.path.dirname(sys.executable) + env["PYTHONHOME"] = os.path.dirname(real) if sysconfig.is_python_build(True): env["PYTHONPATH"] = os.path.dirname(os.__file__) @@ -258,9 +268,7 @@ def get(python, env=None): self.fail('Non-zero return code {0} (0x{0:08X})' .format(p.returncode)) return out, err - real = os.path.realpath(sys.executable) - link = os.path.abspath(TESTFN) - os.symlink(real, link) + try: self.assertEqual(get(real), get(link, env)) finally: diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index e1a4d134e2f4e2..6850b473702abc 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -279,7 +279,12 @@ def test_symlinking(self): # symlinked to 'python3.3' in the env, even when symlinking in # general isn't wanted. if usl: - self.assertTrue(os.path.islink(fn)) + if sys.platform == 'win32' and not os.path.exists(self.exe): + # Symlinking is skipped when our executable is already a + # special app symlink + self.assertFalse(os.path.islink(fn)) + else: + self.assertTrue(os.path.islink(fn)) # If a venv is created from a source build and that venv is used to # run the test, the pyvenv.cfg in the venv created in the test will From 788d2303ff9b0fc4e143bc56ea7f06b8071e1ac5 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 28 Jun 2019 11:22:08 -0700 Subject: [PATCH 12/16] Improve check for unlinkable venv --- Lib/test/test_venv.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 6850b473702abc..8310e557b41eda 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -59,6 +59,12 @@ def setUp(self): self.include = 'include' executable = getattr(sys, '_base_executable', sys.executable) self.exe = os.path.split(executable)[-1] + if (sys.platform == 'win32' + and os.path.lexists(executable) + and not os.path.exists(executable)): + self.cannot_link_exe = True + else: + self.cannot_link_exe = False def tearDown(self): rmtree(self.env_dir) @@ -279,7 +285,7 @@ def test_symlinking(self): # symlinked to 'python3.3' in the env, even when symlinking in # general isn't wanted. if usl: - if sys.platform == 'win32' and not os.path.exists(self.exe): + if self.cannot_link_exe: # Symlinking is skipped when our executable is already a # special app symlink self.assertFalse(os.path.islink(fn)) From 35c099c200e1d9fd247184b136a23b5cc0a31adf Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 28 Jun 2019 16:34:47 -0700 Subject: [PATCH 13/16] Fix handling of symlinked venvs --- PC/python_uwp.cpp | 96 ++++++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 2dfd2256ed6dd0..98d04dfa8ca311 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -64,12 +64,31 @@ get_package_family() return std::wstring(); } +static std::wstring +get_package_home() +{ + try { + const auto package = winrt::Windows::ApplicationModel::Package::Current(); + if (package) { + const auto path = package.InstalledLocation(); + if (path) { + return std::wstring(path.Path()); + } + } + } + catch (...) { + } + + return std::wstring(); +} + static PyStatus set_process_name(PyConfig *config) { - PyStatus status; - std::wstring executable, home; + PyStatus status = PyStatus_Ok(); + std::wstring executable; + const auto home = get_package_home(); const auto family = get_package_family(); if (!family.empty()) { @@ -86,61 +105,54 @@ set_process_name(PyConfig *config) } } - home.resize(MAX_PATH); - while (true) { - DWORD len = GetModuleFileNameW( - NULL, home.data(), (DWORD)home.size()); - if (len == 0) { - home.clear(); - break; - } else if (len == home.size() && - GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - home.resize(len * 2); - } else { - home.resize(len); - size_t bslash = home.find_last_of(L"/\\"); - if (bslash != std::wstring::npos) { - home.erase(bslash); + /* Only use module filename if we don't have a home */ + if (home.empty() && executable.empty()) { + executable.resize(MAX_PATH); + while (true) { + DWORD len = GetModuleFileNameW( + NULL, executable.data(), (DWORD)executable.size()); + if (len == 0) { + executable.clear(); + break; + } else if (len == executable.size() && + GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + executable.resize(len * 2); } else { - home.clear(); + executable.resize(len); + break; } - break; } } - if (executable.empty() && !home.empty()) { - executable = home + L"\\" + PROGNAME; - } - if (!home.empty()) { status = PyConfig_SetString(config, &config->home, home.c_str()); if (PyStatus_Exception(status)) { return status; } } - if (!executable.empty()) { - status = PyConfig_SetString(config, &config->base_executable, executable.c_str()); - if (PyStatus_Exception(status)) { - return status; - } - const wchar_t *launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); - if (launcherPath) { - status = PyConfig_SetString( - config, &config->executable, launcherPath); - /* bpo-35873: Clear the environment variable to avoid it being - * inherited by child processes. */ - _wputenv_s(L"__PYVENV_LAUNCHER__", L""); - } else { - status = PyConfig_SetString( - config, &config->executable, executable.c_str()); - } - if (PyStatus_Exception(status)) { - return status; + const wchar_t *launcherPath = _wgetenv(L"__PYVENV_LAUNCHER__"); + if (launcherPath) { + if (!executable.empty()) { + status = PyConfig_SetString(config, &config->base_executable, + executable.c_str()); + if (PyStatus_Exception(status)) { + return status; + } } + + status = PyConfig_SetString( + config, &config->executable, launcherPath); + + /* bpo-35873: Clear the environment variable to avoid it being + * inherited by child processes. */ + _wputenv_s(L"__PYVENV_LAUNCHER__", L""); + } else if (!executable.empty()) { + status = PyConfig_SetString( + config, &config->executable, executable.c_str()); } - return PyStatus_Ok(); + return status; } int From b0db83e2fae8b0defea4af7477ba1eb253f08b21 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 28 Jun 2019 19:18:02 -0700 Subject: [PATCH 14/16] Ensure PYTHONUSERBASE can be overridden --- PC/python_uwp.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/PC/python_uwp.cpp b/PC/python_uwp.cpp index 98d04dfa8ca311..73e0d82e550310 100644 --- a/PC/python_uwp.cpp +++ b/PC/python_uwp.cpp @@ -184,9 +184,12 @@ wmain(int argc, wchar_t **argv) goto fail; } - _wputenv_s(L"PYTHONUSERBASE", get_user_base().c_str()); + const wchar_t *p = _wgetenv(L"PYTHONUSERBASE"); + if (!p || !*p) { + _wputenv_s(L"PYTHONUSERBASE", get_user_base().c_str()); + } - const wchar_t *p = wcsrchr(argv[0], L'\\'); + p = wcsrchr(argv[0], L'\\'); if (!p) { p = argv[0]; } From f44462b015f05213ac2f008de2b4c5ed4f19eeb6 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 28 Jun 2019 21:36:57 -0700 Subject: [PATCH 15/16] sys._base_executable is now always defined --- Lib/multiprocessing/popen_spawn_win32.py | 3 +-- Lib/test/test_venv.py | 8 ++++---- Lib/venv/__init__.py | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index de4c5ecf1fa083..ea9c555da39a3c 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -22,8 +22,7 @@ def _path_eq(p1, p2): return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2) -WINENV = (hasattr(sys, '_base_executable') and - not _path_eq(sys.executable, sys._base_executable)) +WINENV = not _path_eq(sys.executable, sys._base_executable) def _close_handles(*handles): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 8310e557b41eda..ea016b59ae2b84 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -28,8 +28,8 @@ # Platforms that set sys._base_executable can create venvs from within # another venv, so no need to skip tests that require venv.create(). requireVenvCreate = unittest.skipUnless( - hasattr(sys, '_base_executable') - or sys.prefix == sys.base_prefix, + sys.prefix == sys.base_prefix + or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') def check_output(cmd, encoding=None): @@ -57,7 +57,7 @@ def setUp(self): self.bindir = 'bin' self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' - executable = getattr(sys, '_base_executable', sys.executable) + executable = sys._base_executable self.exe = os.path.split(executable)[-1] if (sys.platform == 'win32' and os.path.lexists(executable) @@ -108,7 +108,7 @@ def test_defaults(self): else: self.assertFalse(os.path.exists(p)) data = self.get_text_file_contents('pyvenv.cfg') - executable = getattr(sys, '_base_executable', sys.executable) + executable = sys._base_executable path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) fn = self.get_env_file(self.bindir, self.exe) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 2189d6ac4f9894..4ab9cc6d6fb2bc 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -112,7 +112,7 @@ def create_if_needed(d): prompt = self.prompt if self.prompt is not None else context.env_name context.prompt = '(%s) ' % prompt create_if_needed(env_dir) - executable = getattr(sys, '_base_executable', sys.executable) + executable = sys._base_executable dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable context.python_dir = dirname From 07f17503a676978f3a7f1394c0f522c808c6539c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sat, 29 Jun 2019 08:55:21 -0700 Subject: [PATCH 16/16] Add test.support.PythonSymlink to encapsulate platform-specific logic for linking sys.executable --- Lib/test/support/__init__.py | 79 ++++++++++++++++++++++++++++++++++++ Lib/test/test_httpservers.py | 7 ++-- Lib/test/test_platform.py | 41 ++----------------- Lib/test/test_sysconfig.py | 48 +++------------------- 4 files changed, 92 insertions(+), 83 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b5538d22fb2ace..19ea9764e96c45 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -12,6 +12,7 @@ import fnmatch import functools import gc +import glob import importlib import importlib.util import io @@ -2500,6 +2501,84 @@ def skip_unless_symlink(test): msg = "Requires functional symlink implementation" return test if ok else unittest.skip(msg)(test) +class PythonSymlink: + """Creates a symlink for the current Python executable""" + def __init__(self, link=None): + self.link = link or os.path.abspath(TESTFN) + self._linked = [] + self.real = os.path.realpath(sys.executable) + self._also_link = [] + + self._env = None + + self._platform_specific() + + def _platform_specific(self): + pass + + if sys.platform == "win32": + def _platform_specific(self): + import _winapi + + if os.path.lexists(self.real) and not os.path.exists(self.real): + # App symlink appears to not exist, but we want the + # real executable here anyway + self.real = _winapi.GetModuleFileName(0) + + dll = _winapi.GetModuleFileName(sys.dllhandle) + src_dir = os.path.dirname(dll) + dest_dir = os.path.dirname(self.link) + self._also_link.append(( + dll, + os.path.join(dest_dir, os.path.basename(dll)) + )) + for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")): + self._also_link.append(( + runtime, + os.path.join(dest_dir, os.path.basename(runtime)) + )) + + self._env = {k.upper(): os.getenv(k) for k in os.environ} + self._env["PYTHONHOME"] = os.path.dirname(self.real) + if sysconfig.is_python_build(True): + self._env["PYTHONPATH"] = os.path.dirname(os.__file__) + + def __enter__(self): + os.symlink(self.real, self.link) + self._linked.append(self.link) + for real, link in self._also_link: + os.symlink(real, link) + self._linked.append(link) + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + for link in self._linked: + try: + os.remove(link) + except IOError as ex: + if verbose: + print("failed to clean up {}: {}".format(link, ex)) + + def _call(self, python, args, env, returncode): + cmd = [python, *args] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + r = p.communicate() + if p.returncode != returncode: + if verbose: + print(repr(r[0])) + print(repr(r[1]), file=sys.stderr) + raise RuntimeError( + 'unexpected return code: {0} (0x{0:08X})'.format(p.returncode)) + return r + + def call_real(self, *args, returncode=0): + return self._call(self.real, args, None, returncode) + + def call_link(self, *args, returncode=0): + return self._call(self.link, args, self._env, returncode) + + _can_xattr = None def can_xattr(): global _can_xattr diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 8357ee9145d7e5..87d4924a34b365 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -610,9 +610,10 @@ def setUp(self): # The shebang line should be pure ASCII: use symlink if possible. # See issue #7668. + self._pythonexe_symlink = None if support.can_symlink(): self.pythonexe = os.path.join(self.parent_dir, 'python') - os.symlink(sys.executable, self.pythonexe) + self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__() else: self.pythonexe = sys.executable @@ -655,8 +656,8 @@ def setUp(self): def tearDown(self): try: os.chdir(self.cwd) - if self.pythonexe != sys.executable: - os.remove(self.pythonexe) + if self._pythonexe_symlink: + self._pythonexe_symlink.__exit__(None, None, None) if self.nocgi_path: os.remove(self.nocgi_path) if self.file1_path: diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 7c8133a9b7a968..8b64923e174c34 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -20,44 +20,9 @@ def test_architecture(self): @support.skip_unless_symlink def test_architecture_via_symlink(self): # issue3762 - if sys.platform == "win32" and not os.path.exists(sys.executable): - # App symlink appears to not exist, but we want the - # real executable here anyway - import _winapi - real = _winapi.GetModuleFileName(0) - else: - real = os.path.realpath(sys.executable) - link = os.path.abspath(support.TESTFN) - os.symlink(real, link) - - # On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at - # so we add the directory to the path, PYTHONHOME and PYTHONPATH. - env = None - if sys.platform == "win32": - env = {k.upper(): os.environ[k] for k in os.environ} - env["PATH"] = "{};{}".format( - os.path.dirname(real), env.get("PATH", "")) - env["PYTHONHOME"] = os.path.dirname(real) - if sysconfig.is_python_build(True): - env["PYTHONPATH"] = os.path.dirname(os.__file__) - - def get(python, env=None): - cmd = [python, '-c', - 'import platform; print(platform.architecture())'] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env) - r = p.communicate() - if p.returncode: - print(repr(r[0])) - print(repr(r[1]), file=sys.stderr) - self.fail('unexpected return code: {0} (0x{0:08X})' - .format(p.returncode)) - return r - - try: - self.assertEqual(get(sys.executable), get(link, env=env)) - finally: - os.remove(link) + with support.PythonSymlink() as py: + cmd = "-c", "import platform; print(platform.architecture())" + self.assertEqual(py.call_real(*cmd), py.call_link(*cmd)) def test_platform(self): for aliased in (False, True): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index d612cdef95598f..44e44bf5ea995f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -6,7 +6,8 @@ from copy import copy from test.support import (import_module, TESTFN, unlink, check_warnings, - captured_stdout, skip_unless_symlink, change_cwd) + captured_stdout, skip_unless_symlink, change_cwd, + PythonSymlink) import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, @@ -232,47 +233,10 @@ def test_get_scheme_names(self): self.assertEqual(get_scheme_names(), wanted) @skip_unless_symlink - def test_symlink(self): - if sys.platform == "win32" and not os.path.exists(sys.executable): - # App symlink appears to not exist, but we want the - # real executable here anyway - import _winapi - real = _winapi.GetModuleFileName(0) - else: - real = os.path.realpath(sys.executable) - link = os.path.abspath(TESTFN) - os.symlink(real, link) - - # On Windows, the EXE needs to know where pythonXY.dll is at so we have - # to add the directory to the path. - env = None - if sys.platform == "win32": - env = {k.upper(): os.environ[k] for k in os.environ} - env["PATH"] = "{};{}".format( - os.path.dirname(real), env.get("PATH", "")) - # Requires PYTHONHOME as well since we locate stdlib from the - # EXE path and not the DLL path (which should be fixed) - env["PYTHONHOME"] = os.path.dirname(real) - if sysconfig.is_python_build(True): - env["PYTHONPATH"] = os.path.dirname(os.__file__) - - # Issue 7880 - def get(python, env=None): - cmd = [python, '-c', - 'import sysconfig; print(sysconfig.get_platform())'] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, env=env) - out, err = p.communicate() - if p.returncode: - print((out, err)) - self.fail('Non-zero return code {0} (0x{0:08X})' - .format(p.returncode)) - return out, err - - try: - self.assertEqual(get(real), get(link, env)) - finally: - unlink(link) + def test_symlink(self): # Issue 7880 + with PythonSymlink() as py: + cmd = "-c", "import sysconfig; print(sysconfig.get_platform())" + self.assertEqual(py.call_real(*cmd), py.call_link(*cmd)) def test_user_similar(self): # Issue #8759: make sure the posix scheme for the users