bpo-37369: Fix initialization of sys members when launched via an app container (GH-14428)

sys._base_executable is now always defined on all platforms, and can be overridden through configuration.
Also adds test.support.PythonSymlink to encapsulate platform-specific logic for symlinking sys.executable

Author: zooba

Include/cpython/initconfig.h

@@ -373,10 +373,11 @@ typedef struct {
                                        module_search_paths_set is equal
                                        to zero. */
 
-    wchar_t *executable;    /* sys.executable */
-    wchar_t *prefix;        /* sys.prefix */
-    wchar_t *base_prefix;   /* sys.base_prefix */
-    wchar_t *exec_prefix;   /* sys.exec_prefix */
+    wchar_t *executable;        /* sys.executable */
+    wchar_t *base_executable;   /* sys._base_executable */
+    wchar_t *prefix;            /* sys.prefix */
+    wchar_t *base_prefix;       /* sys.base_prefix */
+    wchar_t *exec_prefix;       /* sys.exec_prefix */
     wchar_t *base_exec_prefix;  /* sys.base_exec_prefix */
 
     /* --- Parameter only used by Py_Main() ---------- */

Include/internal/pycore_pathconfig.h

@@ -27,6 +27,8 @@ typedef struct _PyPathConfig {
        are ignored when their value are equal to -1 (unset). */
     int isolated;
     int site_import;
+    /* Set when a venv is detected */
+    wchar_t *base_executable;
 } _PyPathConfig;
 
 #define _PyPathConfig_INIT \

Lib/multiprocessing/popen_spawn_win32.py

@@ -22,8 +22,7 @@
 def _path_eq(p1, p2):
     return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
 
-WINENV = (hasattr(sys, '_base_executable') and
-          not _path_eq(sys.executable, sys._base_executable))
+WINENV = not _path_eq(sys.executable, sys._base_executable)
 
 
 def _close_handles(*handles):

Lib/site.py

@@ -459,13 +459,6 @@ def venv(known_paths):
     env = os.environ
     if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
         executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__']
-    elif sys.platform == 'win32' and '__PYVENV_LAUNCHER__' in env:
-        executable = sys.executable
-        import _winapi
-        sys._base_executable = _winapi.GetModuleFileName(0)
-        # bpo-35873: Clear the environment variable to avoid it being
-        # inherited by child processes.
-        del os.environ['__PYVENV_LAUNCHER__']
     else:
         executable = sys.executable
     exe_dir, _ = os.path.split(os.path.abspath(executable))

Lib/test/support/__init__.py

@@ -12,6 +12,7 @@
 import fnmatch
 import functools
 import gc
+import glob
 import importlib
 import importlib.util
 import io
@@ -2500,6 +2501,84 @@ def skip_unless_symlink(test):
     msg = "Requires functional symlink implementation"
     return test if ok else unittest.skip(msg)(test)
 
+class PythonSymlink:
+    """Creates a symlink for the current Python executable"""
+    def __init__(self, link=None):
+        self.link = link or os.path.abspath(TESTFN)
+        self._linked = []
+        self.real = os.path.realpath(sys.executable)
+        self._also_link = []
+
+        self._env = None
+
+        self._platform_specific()
+
+    def _platform_specific(self):
+        pass
+
+    if sys.platform == "win32":
+        def _platform_specific(self):
+            import _winapi
+
+            if os.path.lexists(self.real) and not os.path.exists(self.real):
+                # App symlink appears to not exist, but we want the
+                # real executable here anyway
+                self.real = _winapi.GetModuleFileName(0)
+
+            dll = _winapi.GetModuleFileName(sys.dllhandle)
+            src_dir = os.path.dirname(dll)
+            dest_dir = os.path.dirname(self.link)
+            self._also_link.append((
+                dll,
+                os.path.join(dest_dir, os.path.basename(dll))
+            ))
+            for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")):
+                self._also_link.append((
+                    runtime,
+                    os.path.join(dest_dir, os.path.basename(runtime))
+                ))
+
+            self._env = {k.upper(): os.getenv(k) for k in os.environ}
+            self._env["PYTHONHOME"] = os.path.dirname(self.real)
+            if sysconfig.is_python_build(True):
+                self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
+
+    def __enter__(self):
+        os.symlink(self.real, self.link)
+        self._linked.append(self.link)
+        for real, link in self._also_link:
+            os.symlink(real, link)
+            self._linked.append(link)
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        for link in self._linked:
+            try:
+                os.remove(link)
+            except IOError as ex:
+                if verbose:
+                    print("failed to clean up {}: {}".format(link, ex))
+
+    def _call(self, python, args, env, returncode):
+        cmd = [python, *args]
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE, env=env)
+        r = p.communicate()
+        if p.returncode != returncode:
+            if verbose:
+                print(repr(r[0]))
+                print(repr(r[1]), file=sys.stderr)
+            raise RuntimeError(
+                'unexpected return code: {0} (0x{0:08X})'.format(p.returncode))
+        return r
+
+    def call_real(self, *args, returncode=0):
+        return self._call(self.real, args, None, returncode)
+
+    def call_link(self, *args, returncode=0):
+        return self._call(self.link, args, self._env, returncode)
+
+
 _can_xattr = None
 def can_xattr():
     global _can_xattr

Lib/test/test_embed.py

@@ -362,6 +362,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pythonpath_env': None,
         'home': None,
         'executable': GET_DEFAULT_CONFIG,
+        'base_executable': GET_DEFAULT_CONFIG,
 
         'prefix': GET_DEFAULT_CONFIG,
         'base_prefix': GET_DEFAULT_CONFIG,
@@ -534,14 +535,16 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
             if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
                 expected['stdio_errors'] = 'surrogateescape'
 
+        if sys.platform == 'win32':
+            default_executable = self.test_exe
+        elif expected['program_name'] is not self.GET_DEFAULT_CONFIG:
+            default_executable = os.path.abspath(expected['program_name'])
+        else:
+            default_executable = os.path.join(os.getcwd(), '_testembed')
         if expected['executable'] is self.GET_DEFAULT_CONFIG:
-            if sys.platform == 'win32':
-                expected['executable'] = self.test_exe
-            else:
-                if expected['program_name'] is not self.GET_DEFAULT_CONFIG:
-                    expected['executable'] = os.path.abspath(expected['program_name'])
-                else:
-                    expected['executable'] = os.path.join(os.getcwd(), '_testembed')
+            expected['executable'] = default_executable
+        if expected['base_executable'] is self.GET_DEFAULT_CONFIG:
+            expected['base_executable'] = default_executable
         if expected['program_name'] is self.GET_DEFAULT_CONFIG:
             expected['program_name'] = './_testembed'
 

Lib/test/test_httpservers.py

@@ -610,9 +610,10 @@ def setUp(self):
 
         # The shebang line should be pure ASCII: use symlink if possible.
         # See issue #7668.
+        self._pythonexe_symlink = None
         if support.can_symlink():
             self.pythonexe = os.path.join(self.parent_dir, 'python')
-            os.symlink(sys.executable, self.pythonexe)
+            self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__()
         else:
             self.pythonexe = sys.executable
 
@@ -655,8 +656,8 @@ def setUp(self):
     def tearDown(self):
         try:
             os.chdir(self.cwd)
-            if self.pythonexe != sys.executable:
-                os.remove(self.pythonexe)
+            if self._pythonexe_symlink:
+                self._pythonexe_symlink.__exit__(None, None, None)
             if self.nocgi_path:
                 os.remove(self.nocgi_path)
             if self.file1_path:

Lib/test/test_platform.py

@@ -20,37 +20,9 @@ def test_architecture(self):
 
     @support.skip_unless_symlink
     def test_architecture_via_symlink(self): # issue3762
-        # On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
-        # so we add the directory to the path, PYTHONHOME and PYTHONPATH.
-        env = None
-        if sys.platform == "win32":
-            env = {k.upper(): os.environ[k] for k in os.environ}
-            env["PATH"] = "{};{}".format(
-                os.path.dirname(sys.executable), env.get("PATH", ""))
-            env["PYTHONHOME"] = os.path.dirname(sys.executable)
-            if sysconfig.is_python_build(True):
-                env["PYTHONPATH"] = os.path.dirname(os.__file__)
-
-        def get(python, env=None):
-            cmd = [python, '-c',
-                'import platform; print(platform.architecture())']
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE, env=env)
-            r = p.communicate()
-            if p.returncode:
-                print(repr(r[0]))
-                print(repr(r[1]), file=sys.stderr)
-                self.fail('unexpected return code: {0} (0x{0:08X})'
-                          .format(p.returncode))
-            return r
-
-        real = os.path.realpath(sys.executable)
-        link = os.path.abspath(support.TESTFN)
-        os.symlink(real, link)
-        try:
-            self.assertEqual(get(real), get(link, env=env))
-        finally:
-            os.remove(link)
+        with support.PythonSymlink() as py:
+            cmd = "-c", "import platform; print(platform.architecture())"
+            self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
 
     def test_platform(self):
         for aliased in (False, True):
@@ -275,6 +247,11 @@ def test_libc_ver(self):
            os.path.exists(sys.executable+'.exe'):
             # Cygwin horror
             executable = sys.executable + '.exe'
+        elif sys.platform == "win32" and not os.path.exists(sys.executable):
+            # App symlink appears to not exist, but we want the
+            # real executable here anyway
+            import _winapi
+            executable = _winapi.GetModuleFileName(0)
         else:
             executable = sys.executable
         platform.libc_ver(executable)

Lib/test/test_sysconfig.py

@@ -6,7 +6,8 @@
 from copy import copy
 
 from test.support import (import_module, TESTFN, unlink, check_warnings,
-                          captured_stdout, skip_unless_symlink, change_cwd)
+                          captured_stdout, skip_unless_symlink, change_cwd,
+                          PythonSymlink)
 
 import sysconfig
 from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -232,39 +233,10 @@ def test_get_scheme_names(self):
         self.assertEqual(get_scheme_names(), wanted)
 
     @skip_unless_symlink
-    def test_symlink(self):
-        # On Windows, the EXE needs to know where pythonXY.dll is at so we have
-        # to add the directory to the path.
-        env = None
-        if sys.platform == "win32":
-            env = {k.upper(): os.environ[k] for k in os.environ}
-            env["PATH"] = "{};{}".format(
-                os.path.dirname(sys.executable), env.get("PATH", ""))
-            # Requires PYTHONHOME as well since we locate stdlib from the
-            # EXE path and not the DLL path (which should be fixed)
-            env["PYTHONHOME"] = os.path.dirname(sys.executable)
-            if sysconfig.is_python_build(True):
-                env["PYTHONPATH"] = os.path.dirname(os.__file__)
-
-        # Issue 7880
-        def get(python, env=None):
-            cmd = [python, '-c',
-                   'import sysconfig; print(sysconfig.get_platform())']
-            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                 stderr=subprocess.PIPE, env=env)
-            out, err = p.communicate()
-            if p.returncode:
-                print((out, err))
-                self.fail('Non-zero return code {0} (0x{0:08X})'
-                            .format(p.returncode))
-            return out, err
-        real = os.path.realpath(sys.executable)
-        link = os.path.abspath(TESTFN)
-        os.symlink(real, link)
-        try:
-            self.assertEqual(get(real), get(link, env))
-        finally:
-            unlink(link)
+    def test_symlink(self): # Issue 7880
+        with PythonSymlink() as py:
+            cmd = "-c", "import sysconfig; print(sysconfig.get_platform())"
+            self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
 
     def test_user_similar(self):
         # Issue #8759: make sure the posix scheme for the users

Lib/test/test_venv.py

@@ -28,8 +28,8 @@
 # Platforms that set sys._base_executable can create venvs from within
 # another venv, so no need to skip tests that require venv.create().
 requireVenvCreate = unittest.skipUnless(
-    hasattr(sys, '_base_executable')
-    or sys.prefix == sys.base_prefix,
+    sys.prefix == sys.base_prefix
+    or sys._base_executable != sys.executable,
     'cannot run venv.create from within a venv on this platform')
 
 def check_output(cmd, encoding=None):
@@ -57,8 +57,14 @@ def setUp(self):
             self.bindir = 'bin'
             self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
             self.include = 'include'
-        executable = getattr(sys, '_base_executable', sys.executable)
+        executable = sys._base_executable
         self.exe = os.path.split(executable)[-1]
+        if (sys.platform == 'win32'
+            and os.path.lexists(executable)
+            and not os.path.exists(executable)):
+            self.cannot_link_exe = True
+        else:
+            self.cannot_link_exe = False
 
     def tearDown(self):
         rmtree(self.env_dir)
@@ -102,7 +108,7 @@ def test_defaults(self):
         else:
             self.assertFalse(os.path.exists(p))
         data = self.get_text_file_contents('pyvenv.cfg')
-        executable = getattr(sys, '_base_executable', sys.executable)
+        executable = sys._base_executable
         path = os.path.dirname(executable)
         self.assertIn('home = %s' % path, data)
         fn = self.get_env_file(self.bindir, self.exe)
@@ -158,20 +164,16 @@ def test_prefixes(self):
         """
         Test that the prefix values are as expected.
         """
-        #check our prefixes
-        self.assertEqual(sys.base_prefix, sys.prefix)
-        self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)
-
         # check a venv's prefixes
         rmtree(self.env_dir)
         self.run_with_capture(venv.create, self.env_dir)
         envpy = os.path.join(self.env_dir, self.bindir, self.exe)
         cmd = [envpy, '-c', None]
         for prefix, expected in (
             ('prefix', self.env_dir),
-            ('prefix', self.env_dir),
-            ('base_prefix', sys.prefix),
-            ('base_exec_prefix', sys.exec_prefix)):
+            ('exec_prefix', self.env_dir),
+            ('base_prefix', sys.base_prefix),
+            ('base_exec_prefix', sys.base_exec_prefix)):
             cmd[2] = 'import sys; print(sys.%s)' % prefix
             out, err = check_output(cmd)
             self.assertEqual(out.strip(), expected.encode())
@@ -283,7 +285,12 @@ def test_symlinking(self):
             # symlinked to 'python3.3' in the env, even when symlinking in
             # general isn't wanted.
             if usl:
-                self.assertTrue(os.path.islink(fn))
+                if self.cannot_link_exe:
+                    # Symlinking is skipped when our executable is already a
+                    # special app symlink
+                    self.assertFalse(os.path.islink(fn))
+                else:
+                    self.assertTrue(os.path.islink(fn))
 
     # If a venv is created from a source build and that venv is used to
     # run the test, the pyvenv.cfg in the venv created in the test will