Favor the "venv" sysconfig install scheme over the default and distut…

…ils scheme

Python is preparing to allow re-distributors to set custom sysconfig install schemes in 3.11+:

https://bugs.python.org/issue43976

Fedora is already adapting the default installation scheme to their needs:

https://lists.fedoraproject.org/archives/list/[email protected]/thread/AAGUFQZ4RZDU7KUN4HA43KQJCMSFR3GW/

With either of the above, the distributors need to signalize the paths used
in virtual environments somehow.
When they set the "venv" install scheme in sysconfig,
it is now favored over the default sysconfig scheme as well as over distutils.

A similar technique was proposed to Python,
for the venv module: https://bugs.python.org/issue45413

Fixes #2208

docs/changelog/2208.feature.rst

@@ -0,0 +1,4 @@
+If a ``"venv"`` install scheme exists in ``sysconfig``, virtualenv now uses it to create new virtual environments.
+This allows Python distributors, such as Fedora, to patch/replace the default install scheme without affecting
+the paths in new virtual environments.
+A similar technique `was proposed to Python, for the venv module <https://bugs.python.org/issue45413>`_ - by ``hroncok``

src/virtualenv/discovery/py_info.py

@@ -73,7 +73,18 @@ def abs_path(v):
         self.file_system_encoding = u(sys.getfilesystemencoding())
         self.stdout_encoding = u(getattr(sys.stdout, "encoding", None))
 
-        self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()}
+        if "venv" in sysconfig.get_scheme_names():
+            self.sysconfig_scheme = "venv"
+            self.sysconfig_paths = {
+                u(i): u(sysconfig.get_path(i, expand=False, scheme="venv")) for i in sysconfig.get_path_names()
+            }
+            # we cannot use distutils at all if "venv" exists, distutils don't know it
+            self.distutils_install = {}
+        else:
+            self.sysconfig_scheme = None
+            self.sysconfig_paths = {u(i): u(sysconfig.get_path(i, expand=False)) for i in sysconfig.get_path_names()}
+            self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()}
+
         # https://bugs.python.org/issue22199
         makefile = getattr(sysconfig, "get_makefile_filename", getattr(sysconfig, "_get_makefile_filename", None))
         self.sysconfig = {
@@ -95,7 +106,6 @@ def abs_path(v):
         if self.implementation == "PyPy" and sys.version_info.major == 2:
             self.sysconfig_vars[u"implementation_lower"] = u"python"
 
-        self.distutils_install = {u(k): u(v) for k, v in self._distutils_install().items()}
         confs = {k: (self.system_prefix if v.startswith(self.prefix) else v) for k, v in self.sysconfig_vars.items()}
         self.system_stdlib = self.sysconfig_path("stdlib", confs)
         self.system_stdlib_platform = self.sysconfig_path("platstdlib", confs)
@@ -119,7 +129,7 @@ def _fast_get_system_executable(self):
 
     def install_path(self, key):
         result = self.distutils_install.get(key)
-        if result is None:  # use sysconfig if distutils is unavailable
+        if result is None:  # use sysconfig if sysconfig_scheme is set or distutils is unavailable
             # set prefixes to empty => result is relative from cwd
             prefixes = self.prefix, self.exec_prefix, self.base_prefix, self.base_exec_prefix
             config_var = {k: "" if v in prefixes else v for k, v in self.sysconfig_vars.items()}

tests/unit/discovery/py_info/test_py_info.py

@@ -6,6 +6,7 @@
 import logging
 import os
 import sys
+import sysconfig
 from collections import namedtuple
 from textwrap import dedent
 
@@ -311,3 +312,64 @@ def test_py_info_to_system_raises(session_app_data, mocker, caplog, skip_if_test
     assert log.levelno == logging.INFO
     expected = "ignore {} due cannot resolve system due to RuntimeError('failed to detect ".format(sys.executable)
     assert expected in log.message
+
+
+def _stringify_schemes_dict(schemes_dict):
+    """
+    Since this file has from __future__ import unicode_literals, we manually cast all values of mocked install_schemes
+    to str() as the original schemes are not unicode on Python 2.
+    """
+    return {str(n): {str(k): str(v) for k, v in s.items()} for n, s in schemes_dict.items()}
+
+
+def test_custom_venv_install_scheme_is_prefered(mocker):
+    # The paths in this test are Fedora paths, but we set them for nt as well, so the test also works on Windows,
+    # despite the actual values are nonsense there.
+    # Values were simplified to be compatible with all the supported Python versions.
+    default_scheme = {
+        "stdlib": "{base}/lib/python{py_version_short}",
+        "platstdlib": "{platbase}/lib/python{py_version_short}",
+        "purelib": "{base}/local/lib/python{py_version_short}/site-packages",
+        "platlib": "{platbase}/local/lib/python{py_version_short}/site-packages",
+        "include": "{base}/include/python{py_version_short}",
+        "platinclude": "{platbase}/include/python{py_version_short}",
+        "scripts": "{base}/local/bin",
+        "data": "{base}/local",
+    }
+    venv_scheme = {key: path.replace("local", "") for key, path in default_scheme.items()}
+    sysconfig_install_schemes = {
+        "posix_prefix": default_scheme,
+        "nt": default_scheme,
+        "pypy": default_scheme,
+        "pypy_nt": default_scheme,
+        "venv": venv_scheme,
+    }
+    if getattr(sysconfig, "get_preferred_scheme", None):
+        sysconfig_install_schemes[sysconfig.get_preferred_scheme("prefix")] = default_scheme
+
+    if sys.version_info[0] == 2:
+        sysconfig_install_schemes = _stringify_schemes_dict(sysconfig_install_schemes)
+    mocker.patch("sysconfig._INSTALL_SCHEMES", sysconfig_install_schemes)
+
+    # On Python < 3.10, the distutils schemes are not derived from sysconfig schemes
+    # So we mock them as well to assert the custom "venv" install scheme has priority
+    distutils_scheme = {
+        "purelib": "$base/local/lib/python$py_version_short/site-packages",
+        "platlib": "$platbase/local/lib/python$py_version_short/site-packages",
+        "headers": "$base/include/python$py_version_short/$dist_name",
+        "scripts": "$base/local/bin",
+        "data": "$base/local",
+    }
+    distutils_schemes = {
+        "unix_prefix": distutils_scheme,
+        "nt": distutils_scheme,
+    }
+
+    if sys.version_info[0] == 2:
+        distutils_schemes = _stringify_schemes_dict(distutils_schemes)
+    mocker.patch("distutils.command.install.INSTALL_SCHEMES", distutils_schemes)
+
+    pyinfo = PythonInfo()
+    pyver = "{}.{}".format(pyinfo.version_info.major, pyinfo.version_info.minor)
+    assert pyinfo.install_path("scripts") == "bin"
+    assert pyinfo.install_path("purelib").replace(os.sep, "/") == "lib/python{}/site-packages".format(pyver)