Skip to content

Commit

Permalink
config: introduce installer.no-binary
Browse files Browse the repository at this point in the history
This change replaces the `--no-binary` option introduced in #5600 as
the original implementation could cause inconsistent results when the
add, update or lock commands were used.

This implementation makes use of a new configuration
`installer.no-binary` to allow for user specification of sdist
preference for select packages.
  • Loading branch information
abn committed May 14, 2022
1 parent feb11b1 commit b38709e
Show file tree
Hide file tree
Showing 11 changed files with 168 additions and 90 deletions.
6 changes: 0 additions & 6 deletions docs/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,6 @@ option is used.
* `--default`: Only include the main dependencies. (**Deprecated**)
* `--sync`: Synchronize the environment with the locked packages and the specified groups.
* `--no-root`: Do not install the root package (your project).
* `--no-binary`: Do not use binary distributions for packages matching given policy. Use package name to disallow a specific package; or `:all:` to disallow and `:none:` to force binary for all packages.
* `--dry-run`: Output the operations but do not execute anything (implicitly enables --verbose).
* `--extras (-E)`: Features to install (multiple values allowed).
* `--no-dev`: Do not install dev dependencies. (**Deprecated**)
Expand All @@ -234,11 +233,6 @@ option is used.
When `--only` is specified, `--with` and `--without` options are ignored.
{{% /note %}}

{{% note %}}
The `--no-binary` option will only work with the new installer. For the old installer,
this is ignored.
{{% /note %}}


## update

Expand Down
46 changes: 46 additions & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,52 @@ the number of maximum workers is still limited at `number_of_cores + 4`.
This configuration will be ignored when `installer.parallel` is set to false.
{{% /note %}}

### `installer.no-binary`

**Type**: string | bool

*Introduced in 1.2.0*

When set this configuration allows users to configure package distribution format policy for all or
specific packages.

{{% note %}}
This configuration is only respected when using the new installer. If you have disabled it please
consider re-enabling it.

As with all configurations described here, this is a user specific configuration. This means that this
is not taken into consideration when a lockfile is generated or dependencies are resolved. This is
applied only when selecting which distribution for dependency should be installed into a Poetry managed
environment.
{{% /note %}}

| Configuration | Description |
|------------------------|------------------------------------------------------------|
| `:all:` or `true` | Disallow binary distributions for all packages. |
| `:none:` or `false` | Allow binary distributions for all packages. |
| `package[,package,..]` | Disallow binary distributions for specified packages only. |

For project specific usage, it is recommended that this be configured with the `--local`.

```bash
poetry config --local installer.no-binary :all:
```

{{% note %}}
For CI or container environments using [environment variable](#using-environment-variables)
to configure this might be useful.

```bash
export POETRY_INSTALLER_NO_BINARY=:all:
```
{{% /note %}}

{{% warning %}}
Unless this is required system-wide, if configured globally, you could encounter slower install times
across all your projects if incorrectly set.
{{% /warning %}}


### `virtualenvs.create`

**Type**: boolean
Expand Down
68 changes: 67 additions & 1 deletion src/poetry/config/config.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import dataclasses
import logging
import os
import re
Expand All @@ -11,6 +12,7 @@
from typing import Callable

from poetry.core.toml import TOMLFile
from poetry.core.utils.helpers import canonicalize_name

from poetry.config.dict_config_source import DictConfigSource
from poetry.config.file_config_source import FileConfigSource
Expand All @@ -34,6 +36,67 @@ def int_normalizer(val: str) -> int:
return int(val)


@dataclasses.dataclass
class PackageFilterPolicy:
policy: dataclasses.InitVar[str | list[str] | None]
packages: list[str] = dataclasses.field(init=False)

def __post_init__(self, policy: str | list[str] | None) -> None:
if not policy:
policy = []
elif isinstance(policy, str):
policy = self.normalize(policy)
self.packages = policy

def allows(self, package_name: str) -> bool:
if ":all:" in self.packages:
return False

return (
not self.packages
or ":none:" in self.packages
or canonicalize_name(package_name) not in self.packages
)

@classmethod
def is_reserved(cls, name: str) -> bool:
return bool(re.match(r":(all|none):", name))

@classmethod
def normalize(cls, policy: str) -> list[str]:
if boolean_validator(policy):
if boolean_normalizer(policy):
return [":all:"]
else:
return [":none:"]

return list(
{
name.strip() if cls.is_reserved(name) else canonicalize_name(name)
for name in policy.strip().split(",")
if name
}
)

@classmethod
def validator(cls, policy: str) -> bool:
if boolean_validator(policy):
return True

names = policy.strip().split(",")

for name in names:
if (
not name
or (cls.is_reserved(name) and len(names) == 1)
or re.match(r"^[a-zA-Z\d_-]+$", name)
):
continue
return False

return True


logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -61,7 +124,7 @@ class Config:
"prefer-active-python": False,
},
"experimental": {"new-installer": True, "system-git-client": False},
"installer": {"parallel": True, "max-workers": None},
"installer": {"parallel": True, "max-workers": None, "no-binary": None},
}

def __init__(
Expand Down Expand Up @@ -196,6 +259,9 @@ def _get_normalizer(name: str) -> Callable[[str], Any]:
if name == "installer.max-workers":
return int_normalizer

if name == "installer.no-binary":
return PackageFilterPolicy.normalize

return lambda val: val

@classmethod
Expand Down
6 changes: 6 additions & 0 deletions src/poetry/console/commands/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from cleo.helpers import argument
from cleo.helpers import option

from poetry.config.config import PackageFilterPolicy
from poetry.console.commands.command import Command


Expand Down Expand Up @@ -107,6 +108,11 @@ def unique_config_values(self) -> dict[str, tuple[Any, Any, Any]]:
int_normalizer,
None,
),
"installer.no-binary": (
PackageFilterPolicy.validator,
PackageFilterPolicy.normalize,
None,
),
}

return unique_config_values
Expand Down
21 changes: 0 additions & 21 deletions src/poetry/console/commands/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,6 @@ class InstallCommand(InstallerCommand):
option(
"no-root", None, "Do not install the root package (the current project)."
),
option(
"no-binary",
None,
"Do not use binary distributions for packages matching given policy.\n"
"Use package name to disallow a specific package; or <b>:all:</b> to\n"
"disallow and <b>:none:</b> to force binary for all packages. Multiple\n"
"packages can be specified separated by commas.",
flag=False,
multiple=True,
),
option(
"dry-run",
None,
Expand Down Expand Up @@ -108,17 +98,6 @@ def handle(self) -> int:

with_synchronization = True

if self.option("no-binary"):
policy = ",".join(self.option("no-binary", []))
try:
self._installer.no_binary(policy=policy)
except ValueError as e:
self.line_error(
f"<warning>Invalid value (<c1>{policy}</>) for"
f" `<b>--no-binary</b>`</>.\n\n<error>{e}</>"
)
return 1

self._installer.only_groups(self.activated_groups)
self._installer.dry_run(self.option("dry-run"))
self._installer.requires_synchronization(with_synchronization)
Expand Down
31 changes: 7 additions & 24 deletions src/poetry/installation/chooser.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

from packaging.tags import Tag

from poetry.utils.helpers import canonicalize_name
from poetry.config.config import Config
from poetry.config.config import PackageFilterPolicy
from poetry.utils.patterns import wheel_file_re


Expand Down Expand Up @@ -58,30 +59,12 @@ class Chooser:
A Chooser chooses an appropriate release archive for packages.
"""

def __init__(self, pool: Pool, env: Env) -> None:
def __init__(self, pool: Pool, env: Env, config: Config | None = None) -> None:
self._pool = pool
self._env = env
self._no_binary_policy: set[str] = set()

def set_no_binary_policy(self, policy: str) -> None:
self._no_binary_policy = {
name.strip() if re.match(r":(all|none):", name) else canonicalize_name(name)
for name in policy.split(",")
}

if {":all:", ":none:"} <= self._no_binary_policy:
raise ValueError(
"Ambiguous binary policy containing :all: and :none: given."
)

def allow_binary(self, package_name: str) -> bool:
if ":all:" in self._no_binary_policy:
return False

return (
not self._no_binary_policy
or ":none:" in self._no_binary_policy
or canonicalize_name(package_name) not in self._no_binary_policy
self._config = config or Config.create()
self._no_binary_policy: PackageFilterPolicy = PackageFilterPolicy(
self._config.get("installer.no-binary", [])
)

def choose_for(self, package: Package) -> Link:
Expand All @@ -91,7 +74,7 @@ def choose_for(self, package: Package) -> Link:
links = []
for link in self._get_links(package):
if link.is_wheel:
if not self.allow_binary(package.name):
if not self._no_binary_policy.allows(package.name):
logger.debug(
"Skipping wheel for %s as requested in no binary policy for"
" package (%s)",
Expand Down
5 changes: 1 addition & 4 deletions src/poetry/installation/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def __init__(
self._verbose = False
self._authenticator = Authenticator(config, self._io)
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env)
self._chooser = Chooser(pool, self._env, config)

if parallel is None:
parallel = config.get("installer.parallel", True)
Expand Down Expand Up @@ -92,9 +92,6 @@ def updates_count(self) -> int:
def removals_count(self) -> int:
return self._executed["uninstall"]

def set_no_binary_policy(self, policy: str) -> None:
self._chooser.set_no_binary_policy(policy)

def supports_fancy_output(self) -> bool:
return self._io.output.is_decorated() and not self._dry_run

Expand Down
5 changes: 0 additions & 5 deletions src/poetry/installation/installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,6 @@ def verbose(self, verbose: bool = True) -> Installer:
def is_verbose(self) -> bool:
return self._verbose

def no_binary(self, policy: str) -> Installer:
if self._executor:
self._executor.set_no_binary_policy(policy=policy)
return self

def only_groups(self, groups: Iterable[str]) -> Installer:
self._groups = groups

Expand Down
36 changes: 35 additions & 1 deletion tests/console/commands/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@

import pytest

from deepdiff import DeepDiff
from poetry.core.pyproject.exceptions import PyProjectException

from poetry.config.config_source import ConfigSource
from poetry.factory import Factory
from tests.conftest import Config


if TYPE_CHECKING:
Expand All @@ -20,7 +22,6 @@
from pytest_mock import MockerFixture

from poetry.config.dict_config_source import DictConfigSource
from tests.conftest import Config
from tests.types import CommandTesterFactory
from tests.types import FixtureDirGetter

Expand Down Expand Up @@ -53,6 +54,7 @@ def test_list_displays_default_value_if_not_set(
experimental.new-installer = true
experimental.system-git-client = false
installer.max-workers = null
installer.no-binary = null
installer.parallel = true
virtualenvs.create = true
virtualenvs.in-project = null
Expand Down Expand Up @@ -80,6 +82,7 @@ def test_list_displays_set_get_setting(
experimental.new-installer = true
experimental.system-git-client = false
installer.max-workers = null
installer.no-binary = null
installer.parallel = true
virtualenvs.create = false
virtualenvs.in-project = null
Expand Down Expand Up @@ -131,6 +134,7 @@ def test_list_displays_set_get_local_setting(
experimental.new-installer = true
experimental.system-git-client = false
installer.max-workers = null
installer.no-binary = null
installer.parallel = true
virtualenvs.create = false
virtualenvs.in-project = null
Expand Down Expand Up @@ -200,3 +204,33 @@ def test_config_installer_parallel(
"install"
)._command._installer._executor._max_workers
assert workers == 1


@pytest.mark.parametrize(
("value", "expected"),
[
("true", [":all:"]),
("1", [":all:"]),
("false", [":none:"]),
("0", [":none:"]),
("pytest", ["pytest"]),
("PyTest", ["pytest"]),
("pytest,black", ["pytest", "black"]),
("", []),
],
)
def test_config_installer_no_binary(
tester: CommandTester, value: str, expected: list[str]
) -> None:
setting = "installer.no-binary"

tester.execute(setting)
assert tester.io.fetch_output().strip() == "null"

config = Config.create()
assert not config.get(setting)

tester.execute(f"{setting} '{value}'")

config = Config.create(reload=True)
assert not DeepDiff(config.get(setting), expected, ignore_order=True)
Loading

0 comments on commit b38709e

Please sign in to comment.