From 8c409a185b1a8258c32ba616fe9947043a04d184 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 17 Dec 2021 09:43:05 -0500 Subject: [PATCH 1/8] Extract 'incomplete_scheme' as a variable for readability. --- distutils/command/install.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/distutils/command/install.py b/distutils/command/install.py index 18b352fa..df181102 100644 --- a/distutils/command/install.py +++ b/distutils/command/install.py @@ -445,12 +445,17 @@ def dump_dirs(self, msg): def finalize_unix(self): """Finalizes options for posix platforms.""" if self.install_base is not None or self.install_platbase is not None: - if ((self.install_lib is None and - self.install_purelib is None and - self.install_platlib is None) or + incomplete_scheme = ( + ( + self.install_lib is None and + self.install_purelib is None and + self.install_platlib is None + ) or self.install_headers is None or self.install_scripts is None or - self.install_data is None): + self.install_data is None + ) + if incomplete_scheme: raise DistutilsOptionError( "install-base or install-platbase supplied, but " "installation scheme is incomplete") From da15b6e30ec48a3472b518b63c63ce6363b4e1cd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 17 Dec 2021 10:15:07 -0500 Subject: [PATCH 2/8] Extract _pypy_hack method. --- distutils/command/install.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/distutils/command/install.py b/distutils/command/install.py index df181102..407b96d6 100644 --- a/distutils/command/install.py +++ b/distutils/command/install.py @@ -517,19 +517,20 @@ def finalize_other(self): def select_scheme(self, name): """Sets the install directories by applying the install schemes.""" # it's the caller's problem if they supply a bad name! - if (hasattr(sys, 'pypy_version_info') and - sys.version_info < (3, 8) and - not name.endswith(('_user', '_home'))): - if os.name == 'nt': - name = 'pypy_nt' - else: - name = 'pypy' - scheme = _load_schemes()[name] + scheme = _load_schemes()[self._pypy_hack(name)] for key in SCHEME_KEYS: attrname = 'install_' + key if getattr(self, attrname) is None: setattr(self, attrname, scheme[key]) + @staticmethod + def _pypy_hack(name): + PY37 = sys.version_info < (3, 8) + old_pypy = hasattr(sys, 'pypy_version_info') and PY37 + prefix = not name.endswith(('_user', '_home')) + pypy_name = 'pypy' + '_nt' * (os.name == 'nt') + return pypy_name if old_pypy and prefix else name + def _expand_attrs(self, attrs): for attr in attrs: val = getattr(self, attr) From f32af4e73a3f994f3a76898bb404919c5722508d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 17 Dec 2021 10:13:33 -0500 Subject: [PATCH 3/8] Honor sysconfig.get_preferred_scheme when selecting install schemes. Fixes #76. --- distutils/command/install.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/distutils/command/install.py b/distutils/command/install.py index 407b96d6..80f5f8c7 100644 --- a/distutils/command/install.py +++ b/distutils/command/install.py @@ -81,6 +81,11 @@ 'data' : '{userbase}', } + INSTALL_SCHEMES['osx_framework_user'] = { + 'headers': + '{userbase}/include/{implementation_lower}{py_version_short}{abiflags}/{dist_name}', + } + # The keys to an installation scheme; if any new types of files are to be # installed, be sure to add an entry to every installation scheme above, # and to SCHEME_KEYS here. @@ -515,9 +520,17 @@ def finalize_other(self): "I don't know how to install stuff on '%s'" % os.name) def select_scheme(self, name): + os_name, sep, key = name.partition('_') + try: + resolved = sysconfig.get_preferred_scheme(key) + except Exception: + resolved = self._pypy_hack(name) + return self._select_scheme(resolved) + + def _select_scheme(self, name): """Sets the install directories by applying the install schemes.""" # it's the caller's problem if they supply a bad name! - scheme = _load_schemes()[self._pypy_hack(name)] + scheme = _load_schemes()[name] for key in SCHEME_KEYS: attrname = 'install_' + key if getattr(self, attrname) is None: From 7828197702541840fd61377434711b788de7a076 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sat, 18 Dec 2021 08:15:26 +0100 Subject: [PATCH 4/8] msvc9compiler: Don't raise DistutilsPlatformError on import It gets raised when get_build_version() returns a too low version or when it fails to detect a MSVC version, which for exaple is the case when CPython is built with gcc/clang. In theory this module isn't needed on non-MSVC platforms, but setuptools imports it anyway when monkey patching, which in turn makes setuptools fail. To work around this only check the version when initialising MSVCCompiler(). This also mirrors the behaviour of the newer MSVCCompiler() (in _msvccompiler) which also raises DistutilsPlatformError() only on initialisation in case it can't find the right MSVC. --- distutils/msvc9compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distutils/msvc9compiler.py b/distutils/msvc9compiler.py index a1b3b02f..14d13775 100644 --- a/distutils/msvc9compiler.py +++ b/distutils/msvc9compiler.py @@ -291,8 +291,6 @@ def query_vcvarsall(version, arch="x86"): # More globals VERSION = get_build_version() -if VERSION < 8.0: - raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) # MACROS = MacroExpander(VERSION) class MSVCCompiler(CCompiler) : @@ -339,6 +337,8 @@ def __init__(self, verbose=0, dry_run=0, force=0): def initialize(self, plat_name=None): # multi-init means we would need to check platform same each time... assert not self.initialized, "don't init multiple times" + if self.__version < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % self.__version) if plat_name is None: plat_name = get_platform() # sanity check for platforms to prevent obscure errors later. From 4d4d32a78f0fcfbeff49e271377f735402ba0d74 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sat, 18 Dec 2021 11:18:49 +0100 Subject: [PATCH 5/8] tests: skip uid/gid using tests under Cygwin There is no single user/group with UID/GID=0 under Cygwin unlike Unix. Skip the tests that assume this for now. --- distutils/tests/test_archive_util.py | 3 ++- distutils/tests/test_sdist.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/distutils/tests/test_archive_util.py b/distutils/tests/test_archive_util.py index ce6456dc..0f90ea15 100644 --- a/distutils/tests/test_archive_util.py +++ b/distutils/tests/test_archive_util.py @@ -339,7 +339,7 @@ def test_make_archive_xztar(self): def test_make_archive_owner_group(self): # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support - if UID_GID_SUPPORT: + if UID_GID_SUPPORT and sys.platform != "cygwin": group = grp.getgrgid(0)[0] owner = pwd.getpwuid(0)[0] else: @@ -365,6 +365,7 @@ def test_make_archive_owner_group(self): @unittest.skipUnless(ZLIB_SUPPORT, "Requires zlib") @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") + @unittest.skipUnless(sys.platform != "cygwin", "Cygwin doesn't have UID=0") def test_tarfile_root_owner(self): tmpdir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') diff --git a/distutils/tests/test_sdist.py b/distutils/tests/test_sdist.py index b087a817..21e3974a 100644 --- a/distutils/tests/test_sdist.py +++ b/distutils/tests/test_sdist.py @@ -1,5 +1,6 @@ """Tests for distutils.command.sdist.""" import os +import sys import tarfile import unittest import warnings @@ -441,6 +442,7 @@ def test_manual_manifest(self): @unittest.skipUnless(ZLIB_SUPPORT, "requires zlib") @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") + @unittest.skipUnless(sys.platform != "cygwin", "Cygwin doesn't have UID=0") @unittest.skipIf(find_executable('tar') is None, "The tar command is not found") @unittest.skipIf(find_executable('gzip') is None, From 2a95a48dcee29f2396bee63de34e2ad0ef1247a3 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sat, 18 Dec 2021 09:14:03 +0100 Subject: [PATCH 6/8] CI: add a CI job for testing under Cygwin There are some tests skipped because of missing docutils, but cygwin currently is missing a docutils package for Python 3.9 --- .github/workflows/main.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 947b8551..bd0e1992 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,30 @@ jobs: - name: Run tests run: tox + test_cygwin: + strategy: + matrix: + python: [39] + platform: [windows-latest] + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v2 + - name: Install Cygwin + uses: cygwin/cygwin-install-action@v1 + with: + platform: x86_64 + packages: >- + python${{ matrix.python }}, + python${{ matrix.python }}-devel, + python${{ matrix.python }}-pytest, + gcc-core, + gcc-g++, + ncompress + - name: Run tests + shell: C:\cygwin\bin\env.exe CYGWIN_NOWINPATH=1 CHERE_INVOKING=1 C:\cygwin\bin\bash.exe -leo pipefail -o igncr {0} + run: | + pytest -rs + ci_setuptools: # Integration testing with setuptools strategy: From 629f8ff0517d090dd6931794161d67d64673b016 Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sat, 18 Dec 2021 07:58:00 +0100 Subject: [PATCH 7/8] cygwinccompiler: Split CC env var before passing to subprocess 4113bc31a8e62 added support for clang by respecting the CC env variable. The content of the env var gets passed to check_output() as the compiler executable. But CC is not just a path to an executable but a command line, such as "ccache gcc" which makes checkout_output() fail in those cases because it will try to look for "ccache gcc" in PATH instead of "ccache". To fix the issue use shlex to parse the command line before passing it to check_output(). --- distutils/cygwinccompiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distutils/cygwinccompiler.py b/distutils/cygwinccompiler.py index ad6cc44b..09be5ba2 100644 --- a/distutils/cygwinccompiler.py +++ b/distutils/cygwinccompiler.py @@ -50,6 +50,7 @@ import os import sys import copy +import shlex from subprocess import Popen, PIPE, check_output import re @@ -421,5 +422,5 @@ def get_versions(): def is_cygwincc(cc): '''Try to determine if the compiler that would be used is from cygwin.''' - out_string = check_output([cc, '-dumpmachine']) + out_string = check_output(shlex.split(cc) + ['-dumpmachine']) return out_string.strip().endswith(b'cygwin') From 1d98a156026cbbb43588c5f307a3b8d66d40a1bb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 18 Dec 2021 10:18:55 -0500 Subject: [PATCH 8/8] Extract grp/pwd handling to a unix_compat module. --- distutils/tests/test_archive_util.py | 13 ++++--------- distutils/tests/test_sdist.py | 13 +++---------- distutils/tests/unix_compat.py | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 distutils/tests/unix_compat.py diff --git a/distutils/tests/test_archive_util.py b/distutils/tests/test_archive_util.py index 0f90ea15..c5560372 100644 --- a/distutils/tests/test_archive_util.py +++ b/distutils/tests/test_archive_util.py @@ -14,16 +14,11 @@ from distutils.spawn import find_executable, spawn from distutils.tests import support from test.support import run_unittest, patch +from .unix_compat import require_unix_id, require_uid_0, grp, pwd, UID_0_SUPPORT from .py38compat import change_cwd from .py38compat import check_warnings -try: - import grp - import pwd - UID_GID_SUPPORT = True -except ImportError: - UID_GID_SUPPORT = False try: import zipfile @@ -339,7 +334,7 @@ def test_make_archive_xztar(self): def test_make_archive_owner_group(self): # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support - if UID_GID_SUPPORT and sys.platform != "cygwin": + if UID_0_SUPPORT: group = grp.getgrgid(0)[0] owner = pwd.getpwuid(0)[0] else: @@ -364,8 +359,8 @@ def test_make_archive_owner_group(self): self.assertTrue(os.path.exists(res)) @unittest.skipUnless(ZLIB_SUPPORT, "Requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") - @unittest.skipUnless(sys.platform != "cygwin", "Cygwin doesn't have UID=0") + @require_unix_id + @require_uid_0 def test_tarfile_root_owner(self): tmpdir = self._create_files() base_name = os.path.join(self.mkdtemp(), 'archive') diff --git a/distutils/tests/test_sdist.py b/distutils/tests/test_sdist.py index 21e3974a..880044fa 100644 --- a/distutils/tests/test_sdist.py +++ b/distutils/tests/test_sdist.py @@ -1,6 +1,5 @@ """Tests for distutils.command.sdist.""" import os -import sys import tarfile import unittest import warnings @@ -8,6 +7,7 @@ from os.path import join from textwrap import dedent from test.support import captured_stdout, run_unittest +from .unix_compat import require_unix_id, require_uid_0, pwd, grp from .py38compat import check_warnings @@ -17,13 +17,6 @@ except ImportError: ZLIB_SUPPORT = False -try: - import grp - import pwd - UID_GID_SUPPORT = True -except ImportError: - UID_GID_SUPPORT = False - from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution from distutils.tests.test_config import BasePyPIRCCommandTestCase @@ -441,8 +434,8 @@ def test_manual_manifest(self): 'fake-1.0/README.manual']) @unittest.skipUnless(ZLIB_SUPPORT, "requires zlib") - @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") - @unittest.skipUnless(sys.platform != "cygwin", "Cygwin doesn't have UID=0") + @require_unix_id + @require_uid_0 @unittest.skipIf(find_executable('tar') is None, "The tar command is not found") @unittest.skipIf(find_executable('gzip') is None, diff --git a/distutils/tests/unix_compat.py b/distutils/tests/unix_compat.py new file mode 100644 index 00000000..b7718c26 --- /dev/null +++ b/distutils/tests/unix_compat.py @@ -0,0 +1,16 @@ +import sys +import unittest + +try: + import grp + import pwd +except ImportError: + grp = pwd = None + + +UNIX_ID_SUPPORT = grp and pwd +UID_0_SUPPORT = UNIX_ID_SUPPORT and sys.platform != "cygwin" + +require_unix_id = unittest.skipUnless( + UNIX_ID_SUPPORT, "Requires grp and pwd support") +require_uid_0 = unittest.skipUnless(UID_0_SUPPORT, "Requires UID 0 support")