bpo-46890: Fix setting of sys._base_executable with framework builds on macOS (GH-31958)

The side effect of this bug was that venv environments directly
used the main interpreter instead of the intermediate stub executable,
which can cause problems when a script uses system APIs that
require the use of an application bundle.

Lib/test/test_getpath.py

@@ -446,6 +446,182 @@ class MockGetPathTests(unittest.TestCase):
         actual = getpath(ns, expected)
         self.assertEqual(expected, actual)
 
+    def test_framework_macos(self):
+        """ Test framework layout on macOS
+
+        This layout is primarily detected using a compile-time option
+        (WITH_NEXT_FRAMEWORK).
+        """
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python",
+            WITH_NEXT_FRAMEWORK=1,
+            PREFIX="/Library/Frameworks/Python.framework/Versions/9.8",
+            EXEC_PREFIX="/Library/Frameworks/Python.framework/Versions/9.8",
+            ENV___PYVENV_LAUNCHER__="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8",
+            real_executable="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python",
+            library="/Library/Frameworks/Python.framework/Versions/9.8/Python",
+        )
+        ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python")
+        ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8")
+        ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload")
+        ns.add_known_file("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py")
+
+        # This is definitely not the stdlib (see discusion in bpo-46890)
+        #ns.add_known_file("/Library/Frameworks/lib/python98.zip")
+
+        expected = dict(
+            executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8",
+            prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8",
+            base_prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/Library/Frameworks/Python.framework/Versions/9.8/lib/python98.zip",
+                "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8",
+                "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_alt_framework_macos(self):
+        """ Test framework layout on macOS with alternate framework name
+
+        ``--with-framework-name=DebugPython``
+
+        This layout is primarily detected using a compile-time option
+        (WITH_NEXT_FRAMEWORK).
+        """
+        ns = MockPosixNamespace(
+            argv0="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython",
+            os_name="darwin",
+            WITH_NEXT_FRAMEWORK=1,
+            PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            EXEC_PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            ENV___PYVENV_LAUNCHER__="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8",
+            real_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython",
+            library="/Library/Frameworks/DebugPython.framework/Versions/9.8/DebugPython",
+            PYTHONPATH=None,
+            ENV_PYTHONHOME=None,
+            ENV_PYTHONEXECUTABLE=None,
+            executable_dir=None,
+            py_setpath=None,
+        )
+        ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython")
+        ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8")
+        ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload")
+        ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py")
+
+        # This is definitely not the stdlib (see discusion in bpo-46890)
+        #ns.add_known_xfile("/Library/lib/python98.zip")
+        expected = dict(
+            executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8",
+            prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8",
+            base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python98.zip",
+                "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8",
+                "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_framework_macos(self):
+        """Test a venv layout on macOS using a framework build
+        """
+        venv_path = "/tmp/workdir/venv"
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python",
+            WITH_NEXT_FRAMEWORK=1,
+            PREFIX="/Library/Frameworks/Python.framework/Versions/9.8",
+            EXEC_PREFIX="/Library/Frameworks/Python.framework/Versions/9.8",
+            ENV___PYVENV_LAUNCHER__=f"{venv_path}/bin/python",
+            real_executable="/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python",
+            library="/Library/Frameworks/Python.framework/Versions/9.8/Python",
+        )
+        ns.add_known_dir(venv_path)
+        ns.add_known_dir(f"{venv_path}/bin")
+        ns.add_known_dir(f"{venv_path}/lib")
+        ns.add_known_dir(f"{venv_path}/lib/python9.8")
+        ns.add_known_xfile(f"{venv_path}/bin/python")
+        ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/Python")
+        ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8")
+        ns.add_known_dir("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload")
+        ns.add_known_xfile("/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/os.py")
+        ns.add_known_file(f"{venv_path}/pyvenv.cfg", [
+            "home = /Library/Frameworks/Python.framework/Versions/9.8/bin"
+        ])
+        expected = dict(
+            executable=f"{venv_path}/bin/python",
+            prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            base_executable="/Library/Frameworks/Python.framework/Versions/9.8/bin/python9.8",
+            base_prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            base_exec_prefix="/Library/Frameworks/Python.framework/Versions/9.8",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/Library/Frameworks/Python.framework/Versions/9.8/lib/python98.zip",
+                "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8",
+                "/Library/Frameworks/Python.framework/Versions/9.8/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
+    def test_venv_alt_framework_macos(self):
+        """Test a venv layout on macOS using a framework build
+
+        ``--with-framework-name=DebugPython``
+        """
+        venv_path = "/tmp/workdir/venv"
+        ns = MockPosixNamespace(
+            os_name="darwin",
+            argv0="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython",
+            WITH_NEXT_FRAMEWORK=1,
+            PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            EXEC_PREFIX="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            ENV___PYVENV_LAUNCHER__=f"{venv_path}/bin/python",
+            real_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython",
+            library="/Library/Frameworks/DebugPython.framework/Versions/9.8/DebugPython",
+        )
+        ns.add_known_dir(venv_path)
+        ns.add_known_dir(f"{venv_path}/bin")
+        ns.add_known_dir(f"{venv_path}/lib")
+        ns.add_known_dir(f"{venv_path}/lib/python9.8")
+        ns.add_known_xfile(f"{venv_path}/bin/python")
+        ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/Resources/Python.app/Contents/MacOS/DebugPython")
+        ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8")
+        ns.add_known_dir("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload")
+        ns.add_known_xfile("/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/os.py")
+        ns.add_known_file(f"{venv_path}/pyvenv.cfg", [
+            "home = /Library/Frameworks/DebugPython.framework/Versions/9.8/bin"
+        ])
+        expected = dict(
+            executable=f"{venv_path}/bin/python",
+            prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            base_executable="/Library/Frameworks/DebugPython.framework/Versions/9.8/bin/python9.8",
+            base_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            base_exec_prefix="/Library/Frameworks/DebugPython.framework/Versions/9.8",
+            module_search_paths_set=1,
+            module_search_paths=[
+                "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python98.zip",
+                "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8",
+                "/Library/Frameworks/DebugPython.framework/Versions/9.8/lib/python9.8/lib-dynload",
+            ],
+        )
+        actual = getpath(ns, expected)
+        self.assertEqual(expected, actual)
+
     def test_venv_macos(self):
         """Test a venv layout on macOS.
 
@@ -787,6 +963,7 @@ class MockPosixNamespace(dict):
         self["config"] = DEFAULT_CONFIG.copy()
         self["os_name"] = "posix"
         self["PLATLIBDIR"] = "lib"
+        self["WITH_NEXT_FRAMEWORK"] = 0
         super().__init__(*a, **kw)
         if argv0:
             self["config"]["orig_argv"] = [argv0]

Makefile.pre.in

@@ -1218,6 +1218,7 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h M
 		-DVERSION='"$(VERSION)"' \
 		-DVPATH='"$(VPATH)"' \
 		-DPLATLIBDIR='"$(PLATLIBDIR)"' \
+		-DPYTHONFRAMEWORK='"$(PYTHONFRAMEWORK)"' \
 		-o $@ $(srcdir)/Modules/getpath.c
 
 Programs/python.o: $(srcdir)/Programs/python.c

Misc/NEWS.d/next/macOS/2022-03-17-09-55-02.bpo-46890.GX-3OO.rst

@@ -0,0 +1,3 @@
+Fix a regression in the setting of ``sys._base_executable`` in framework
+builds, and thereby fix a regression in :mod:`venv` virtual environments
+with such builds.

Modules/getpath.c

@@ -875,6 +875,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
         !decode_to_dict(dict, "os_name", "darwin") ||
 #else
         !decode_to_dict(dict, "os_name", "posix") ||
+#endif
+#ifdef WITH_NEXT_FRAMEWORK
+        !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) ||
+#else
+        !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 0) ||
 #endif
         !decode_to_dict(dict, "PREFIX", PREFIX) ||
         !decode_to_dict(dict, "EXEC_PREFIX", EXEC_PREFIX) ||
@@ -943,3 +948,4 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config)
 
     return _PyStatus_OK();
 }
+

Modules/getpath.py

@@ -33,6 +33,7 @@
 # PREFIX            -- [in] sysconfig.get_config_var(...)
 # EXEC_PREFIX       -- [in] sysconfig.get_config_var(...)
 # PYTHONPATH        -- [in] sysconfig.get_config_var(...)
+# WITH_NEXT_FRAMEWORK   -- [in] sysconfig.get_config_var(...)
 # VPATH             -- [in] sysconfig.get_config_var(...)
 # PLATLIBDIR        -- [in] sysconfig.get_config_var(...)
 # PYDEBUGEXT        -- [in, opt] '_d' on Windows for debug builds
@@ -301,9 +302,19 @@ if ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__:
     # If set, these variables imply that we should be using them as
     # sys.executable and when searching for venvs. However, we should
     # use the argv0 path for prefix calculation
-    base_executable = executable
+
+    if os_name == 'darwin' and WITH_NEXT_FRAMEWORK:
+        # In a framework build the binary in {sys.exec_prefix}/bin is
+        # a stub executable that execs the real interpreter in an
+        # embedded app bundle. That bundle is an implementation detail
+        # and should not affect base_executable.
+        base_executable = f"{dirname(library)}/bin/python{VERSION_MAJOR}.{VERSION_MINOR}"
+    else:
+        base_executable = executable
+
     if not real_executable:
-        real_executable = executable
+        real_executable = base_executable
+        #real_executable_dir = dirname(real_executable)
     executable = ENV_PYTHONEXECUTABLE or ENV___PYVENV_LAUNCHER__
     executable_dir = dirname(executable)