Favor the "venv" sysconfig install scheme over the default and distut…

…ils scheme

Python is preparing to allow re-distributors to set custom sysconfig install schemes in 3.11+:

https://bugs.python.org/issue43976

Fedora is already adapting the default installation scheme to their needs:

https://lists.fedoraproject.org/archives/list/[email protected]/thread/AAGUFQZ4RZDU7KUN4HA43KQJCMSFR3GW/

With either of the above, the distributors need to signalize the paths used in virtual environments somehow. When they set the "venv" install scheme in sysconfig, it is now favored over the default sysconfig scheme as well as over distutils.

Fixes #2208

src/virtualenv/discovery/py_info.py

@@ -73,7 +73,19 @@ def abs_path(v):
         self.file_system_encoding = u(sys.getfilesystemencoding())
         self.stdout_encoding = u(getattr(sys.stdout, "encoding", None))
 
-        self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()}
+        # https://github.com/pypa/virtualenv/issues/2208
+        if "venv" in sysconfig.get_scheme_names():
+            self.sysconfig_scheme = "venv"
+            self.sysconfig_paths = {
+                u(i): u(sysconfig.get_path(i, expand=False, scheme="venv")) for i in sysconfig.get_path_names()
+            }
+            # we cannot use distutils at all if "venv" exists, distutils don't know it
+            self.distutils_install = {}
+        else:
+            self.sysconfig_scheme = None
+            self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()}
+            self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()}
+
         # https://bugs.python.org/issue22199
         makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None))
         self.sysconfig = {
@@ -95,7 +107,6 @@ def abs_path(v):
         if self.implementation == "PyPy" and sys.version_info.major == 2:
             self.sysconfig_vars[u"implementation_lower"] = u"python"
 
-        self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()}
         confs = {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}
         self.system_stdlib = self.sysconfig_path("stdlib", confs)
         self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs)
@@ -119,7 +130,7 @@ def _fast_get_system_executable(self):
 
     def install_path(self, key):
         result = self.distutils_install.get(key)
-        if result is None:  # use sysconfig if distutils is unavailable
+        if result is None:  # use sysconfig if sysconfig_scheme is set or distutils is unavailable
             # set prefixes to empty => result is relative from cwd
             prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix
             config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()}

tests/unit/discovery/py_info/test_py_info.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import sys
+import sysconfig
 from collections import namedtuple
 from textwrap import dedent
 
@@ -311,3 +312,48 @@ def test_py_info_to_system_raises(session_app_data, mocker, caplog, skip_if_test
     assert log.levelno == logging.INFO
     expected = "ignore {} due cannot resolve system due to RuntimeError('failed to detect ".format(sys.executable)
     assert expected in log.message
+
+
+def test_custom_venv_install_scheme_is_prefered(mocker):
+    # The paths in this test are Fedora paths, but we set them for nt as well,
+    # so the test also works on Windows, despite the actual values are nonsense there.
+    # Values were simplified to be compatible with all the supported Python versions.
+    default_scheme = {
+        str("stdlib"): str("{base}/lib/python{py_version_short}"),
+        str("platstdlib"): str("{platbase}/lib/python{py_version_short}"),
+        str("purelib"): str("{base}/local/lib/python{py_version_short}/site-packages"),
+        str("platlib"): str("{platbase}/local/lib/python{py_version_short}/site-packages"),
+        str("include"): str("{base}/include/python{py_version_short}"),
+        str("platinclude"): str("{platbase}/include/python{py_version_short}"),
+        str("scripts"): str("{base}/local/bin"),
+        str("data"): str("{base}/local"),
+    }
+    venv_scheme = {key: path.replace(str("local"), str()) for key, path in default_scheme.items()}
+    sysconfig_install_schemes = {
+        str("posix_prefix"): default_scheme,
+        str("nt"): default_scheme,
+        str("venv"): venv_scheme,
+    }
+    if getattr(sysconfig, "get_preferred_scheme", None):
+        sysconfig_install_schemes[sysconfig.get_preferred_scheme("prefix")] = default_scheme
+    mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes)
+
+    # On Python < 3.10, the distutils schemes are not derived from sysconfig schemes
+    # So we mock them as well to assert the custom "venv" install scheme has priority
+    distutils_scheme = {
+        str("purelib"): str("$base/local/lib/python$py_version_short/site-packages"),
+        str("platlib"): str("$platbase/local/lib/python$py_version_short/site-packages"),
+        str("headers"): str("$base/include/python$py_version_short/$dist_name"),
+        str("scripts"): str("$base/local/bin"),
+        str("data"): str("$base/local"),
+    }
+    distutils_schemes = {
+        str("unix_prefix"): distutils_scheme,
+        str("nt"): distutils_scheme,
+    }
+    mocker.patch("distutils.command.install.INSTALL_SCHEMES", distutils_schemes)
+
+    pyinfo = PythonInfo()
+    pyver = "{}.{}".format(pyinfo.version_info.major, pyinfo.version_info.minor)
+    assert pyinfo.install_path("scripts") == "bin"
+    assert pyinfo.install_path("purelib").replace(os.sep, "/") == "lib/python{}/site-packages".format(pyver)