diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e166608..4ee70b04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,24 +61,15 @@ jobs: pip-version: >- ${{ fromJSON( - github.job_workflow_sha + inputs.cpython-pip-version && inputs.cpython-pip-version - || '["latest", "previous"]' + || '["supported", "lowest"]' ) }} - include: - - os: Ubuntu - python-version: >- - ${{ - github.job_workflow_sha - && '3.12-dev' - || '3.8' - }} - pip-version: main env: TOXENV: >- pip${{ matrix.pip-version }}${{ - !github.job_workflow_sha + !inputs.cpython-pip-version && '-coverage' || '' }} @@ -121,7 +112,7 @@ jobs: run: tox --skip-pkg-install - name: Upload coverage to Codecov if: >- - !github.job_workflow_sha + !inputs.cpython-pip-version uses: codecov/codecov-action@v3 with: files: ./coverage.xml @@ -148,9 +139,9 @@ jobs: - MacOS - Windows python-version: - - pypy-3.8 + - pypy-3.9 pip-version: - - latest + - supported env: TOXENV: pip${{ matrix.pip-version }} steps: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index b024ea8c..23b0a218 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -11,4 +11,4 @@ jobs: uses: ./.github/workflows/ci.yml with: cpython-pip-version: >- - ["main", "latest", "previous"] + ["main", "latest", "supported", "lowest"] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0aa16adf..f9a6a88b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: - build==1.0.0 - pyproject_hooks==1.0.0 - pytest==7.4.2 - language_version: python3.8 + language_version: python3.9 - repo: https://github.com/PyCQA/bandit rev: 1.7.8 hooks: diff --git a/piptools/build.py b/piptools/build.py index c19e206b..3e7a5b2c 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -2,6 +2,7 @@ import collections import contextlib +import os import pathlib import sys import tempfile @@ -119,6 +120,7 @@ def build_project_metadata( src_file: pathlib.Path, build_targets: tuple[str, ...], *, + upgrade_packages: tuple[str, ...] | None = None, attempt_static_parse: bool, isolated: bool, quiet: bool, @@ -159,7 +161,12 @@ def build_project_metadata( return project_metadata src_dir = src_file.parent - with _create_project_builder(src_dir, isolated=isolated, quiet=quiet) as builder: + with _create_project_builder( + src_dir, + upgrade_packages=upgrade_packages, + isolated=isolated, + quiet=quiet, + ) as builder: metadata = _build_project_wheel_metadata(builder) extras = tuple(metadata.get_all("Provides-Extra") or ()) requirements = tuple( @@ -180,9 +187,80 @@ def build_project_metadata( ) +@contextlib.contextmanager +def _env_var( + env_var_name: str, + env_var_value: str, + /, +) -> Iterator[None]: + sentinel = object() + original_pip_constraint = os.getenv(env_var_name, sentinel) + pip_constraint_was_unset = original_pip_constraint is sentinel + + os.environ[env_var_name] = env_var_value + try: + yield + finally: + if pip_constraint_was_unset: + del os.environ[env_var_name] + return + + # Assert here is necessary because MyPy can't infer type + # narrowing in the complex case. + assert isinstance(original_pip_constraint, str) + os.environ[env_var_name] = original_pip_constraint + + +@contextlib.contextmanager +def _temporary_constraints_file_set_for_pip( + upgrade_packages: tuple[str, ...], +) -> Iterator[None]: + with tempfile.NamedTemporaryFile( + mode="w+t", + delete=False, # FIXME: switch to `delete_on_close` in Python 3.12+ + ) as tmpfile: + # NOTE: `delete_on_close=False` here (or rather `delete=False`, + # NOTE: temporarily) is important for cross-platform execution. It is + # NOTE: required on Windows so that the underlying `pip install` + # NOTE: invocation by pypa/build will be able to access the constraint + # NOTE: file via a subprocess and not fail installing it due to a + # NOTE: permission error related to this file handle still open in our + # NOTE: parent process. To achieve this, we `.close()` the file + # NOTE: descriptor before we hand off the control to the build frontend + # NOTE: and with `delete_on_close=False`, the + # NOTE: `tempfile.NamedTemporaryFile()` context manager does not remove + # NOTE: it from disk right away. + # NOTE: Due to support of versions below Python 3.12, we are forced to + # NOTE: temporarily resort to using `delete=False`, meaning that the CM + # NOTE: never attempts removing the file from disk, not even on exit. + # NOTE: So we do this manually until we can migrate to using the more + # NOTE: ergonomic argument `delete_on_close=False`. + + # Write packages to upgrade to a temporary file to set as + # constraints for the installation to the builder environment, + # in case build requirements are among them + tmpfile.write("\n".join(upgrade_packages)) + + # FIXME: replace `delete` with `delete_on_close` in Python 3.12+ + # FIXME: and replace `.close()` with `.flush()` + tmpfile.close() + + try: + with _env_var("PIP_CONSTRAINT", tmpfile.name): + yield + finally: + # FIXME: replace `delete` with `delete_on_close` in Python 3.12+ + # FIXME: and drop this manual deletion + os.unlink(tmpfile.name) + + @contextlib.contextmanager def _create_project_builder( - src_dir: pathlib.Path, *, isolated: bool, quiet: bool + src_dir: pathlib.Path, + *, + upgrade_packages: tuple[str, ...] | None = None, + isolated: bool, + quiet: bool, ) -> Iterator[build.ProjectBuilder]: if quiet: runner = pyproject_hooks.quiet_subprocess_runner @@ -193,7 +271,13 @@ def _create_project_builder( yield build.ProjectBuilder(src_dir, runner=runner) return - with build.env.DefaultIsolatedEnv() as env: + maybe_pip_constrained_context = ( + contextlib.nullcontext() + if upgrade_packages is None + else _temporary_constraints_file_set_for_pip(upgrade_packages) + ) + + with maybe_pip_constrained_context, build.env.DefaultIsolatedEnv() as env: builder = build.ProjectBuilder.from_isolated_env(env, src_dir, runner) env.install(builder.build_system_requires) env.install(builder.get_requires_for_build("wheel")) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 96aa9fa9..2b4522eb 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -370,6 +370,7 @@ def cli( metadata = build_project_metadata( src_file=Path(src_file), build_targets=build_deps_targets, + upgrade_packages=upgrade_packages, attempt_static_parse=not bool(build_deps_targets), isolated=build_isolation, quiet=log.verbosity <= 0, diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index c5031fc2..ad66dc3a 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -3431,7 +3431,6 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv """ ) ) - (tmp_path / "constraints.txt").write_text("wheel<0.43") out = runner.invoke( cli, [ @@ -3446,8 +3445,6 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv "--find-links", os.fspath(MINIMAL_WHEELS_PATH), os.fspath(tmp_path / "pyproject.toml"), - "--constraint", - os.fspath(tmp_path / "constraints.txt"), "--output-file", "-", ], @@ -3455,6 +3452,75 @@ def test_compile_recursive_extras_build_targets(runner, tmp_path, current_resolv expected = rf"""foo[footest] @ {tmp_path.as_uri()} small-fake-a==0.2 small-fake-b==0.3 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +""" + try: + assert out.exit_code == 0 + assert expected == out.stdout + except Exception: # pragma: no cover + print(out.stdout) + print(out.stderr) + raise + + +@backtracking_resolver_only +def test_compile_build_targets_setuptools_no_wheel_dep( + runner, + tmp_path, + current_resolver, +): + """Check that user requests apply to build dependencies. + + This verifies that build deps compilation would not use the latest version + of an unconstrained build requirements list, when the user requested + restricting them. + + It is implemented against `setuptools < 70.1.0` which is known to inject a + dependency on `wheel` (newer `setuptools` vendor it). The idea is that + `pyproject.toml` does not have an upper bound for `setuptools` but the CLI + arg does. And when this works correctly, the `wheel` entry will be included + into the resolved output. + + This is a regression test for + https://github.com/jazzband/pip-tools/pull/1681#issuecomment-2212541289. + """ + (tmp_path / "pyproject.toml").write_text( + dedent( + """ + [project] + name = "foo" + version = "0.0.1" + dependencies = ["small-fake-a"] + """ + ) + ) + (tmp_path / "constraints.txt").write_text("wheel<0.43") + out = runner.invoke( + cli, + [ + "--build-isolation", + "--no-header", + "--no-annotate", + "--no-emit-options", + "--extra", + "dev", + "--build-deps-for", + "wheel", + "--find-links", + os.fspath(MINIMAL_WHEELS_PATH), + os.fspath(tmp_path / "pyproject.toml"), + "--constraint", + os.fspath(tmp_path / "constraints.txt"), + "--upgrade-package", + "setuptools < 70.1.0", # setuptools>=70.1.0 doesn't require wheel any more + "--output-file", + "-", + ], + catch_exceptions=True, + ) + expected = r"""small-fake-a==0.2 wheel==0.42.0 # The following packages are considered to be unsafe in a requirements file: diff --git a/tox.ini b/tox.ini index bb740ee3..85f3dc68 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = # NOTE: keep this in sync with the env list in .github/workflows/ci.yml. - py{38,39,310,311,312,py3}-pip{previous,latest,main}-coverage - pip{previous,latest,main}-coverage - pip{previous,latest,main} + py{38,39,310,311,312,py3}-pip{supported,lowest,latest,main}-coverage + pip{supported,lowest,latest,main}-coverage + pip{supported,lowest,latest,main} checkqa readme skip_missing_interpreters = True @@ -14,7 +14,8 @@ extras = testing coverage: coverage deps = - pipprevious: pip==22.2.* + pipsupported: pip==24.2 + piplowest: pip==22.2.* piplatest: pip pipmain: https://github.com/pypa/pip/archive/main.zip setenv =