diff --git a/README.rst b/README.rst index 2a178e8..c81c701 100644 --- a/README.rst +++ b/README.rst @@ -2,22 +2,59 @@ Introduction ============ Tools to make managing Plone core releases easier. +It is a wrapper around ``zest.releaser``, plus it adds some commands. + +WARNING: this package is only meant for development of core Plone. +It may have useful bits for others, but we may drop features and edge cases at any time if it is no longer useful for Plone. + Installation ------------ -To install plone.releaser add it to your buildout:: +Do ``pip install plone.releaser`` or add it to your buildout:: [release] recipe = zc.recipe.egg - eggs = plone.releaser + eggs = plone.releaser + + +Main usage: release a package +----------------------------- + +In the `Plone core development buildout `_ go to ``src/some.package`` and run ``../../bin/fullrelease``. +This calls the ``fullrelease`` command from ``zest.releaser``, but with some extra hooks and packages available. + +One nice thing it does: look in the checkouts and sources of your ``buildout.coredev`` checkout and update them: +remove the released package from the checkouts and update the version. + +If you are working on branch 6.1 of coredev, then we check if you also have branches 5.2 and 6.0 checked out. +There we check if the branch of the released package is in the sources. +If you make a release from package branch ``main`` and this is the branch used in the sources, then we update the checkouts and sources of this coredev branch as well. + +After releasing a package, you should wait a few minutes before you manually push the changes to all coredev branches. +This gives the PyPI mirrors time to catch up so the new release is available, so Jenkins and GitHub Actions can find it. + -To make it available in buildout.coredev, run buildout with releaser.cfg:: +Main commands +------------- - $ bin/buildout -c releaser.cfg +Take several Buildout files and create pip/mxdev files out of them:: -Usage ------ + $ bin/manage buildout2pip + +Take a Buildout versions file and create a pip constraints file out of it. + + $ bin/manage versions2constraints + +Generate a changelog with changes from all packages since a certain Plone release:: + + $ bin/manage changelog --start=6.1.0a1 + + +Other commands +-------------- + +Some commands are not used much (by Maurits anyway) because they are less needed these days. Check PyPi access to all Plone packages for a certain user:: @@ -35,10 +72,6 @@ Pulls:: $ bin/manage pulls -Changelog:: - - $ bin/manage changelog - Check checkout:: $ bin/manage check-checkout diff --git a/news/72.feature b/news/72.feature new file mode 100644 index 0000000..be1a4ef --- /dev/null +++ b/news/72.feature @@ -0,0 +1,2 @@ +Add buildout2pip manage command. +[maurits] diff --git a/plone/releaser/base.py b/plone/releaser/base.py index a401a2b..2a7f8b1 100644 --- a/plone/releaser/base.py +++ b/plone/releaser/base.py @@ -54,3 +54,96 @@ def add(self, package_name): def remove(self, package_name): return self.__delitem__(package_name) + + +class Source: + """Source definition for mr.developer or mxdev""" + + def __init__( + self, + name="", + protocol=None, + url=None, + pushurl=None, + branch=None, + path=None, + egg=True, + ): + self.name = name + self.protocol = protocol + self.url = url + self.pushurl = pushurl + self.branch = branch + # mxdev has target (default: sources) instead of path (default: src). + self.path = path + # egg=True: mxdev install-mode="direct" + # egg=False: mxdev install-mode="skip" + self.egg = egg + + @classmethod + def create_from_string(cls, name, source_string): + line_options = source_string.split() + protocol = line_options.pop(0) + url = line_options.pop(0) + # September 2023: mr.developer defaults to master, mxdev to main. + options = {"name": name, "protocol": protocol, "url": url, "branch": "master"} + + # The rest of the line options are key/value pairs. + for param in line_options: + if param is not None: + key, value = param.split("=") + if key == "egg": + if value.lower() in ("true", "yes", "on"): + value = True + elif value.lower() in ("false", "no", "off"): + value = False + options[key] = value + return cls(**options) + + @classmethod + def create_from_section(cls, section): + options = { + "name": section.name, + "protocol": section.get("cvs", "git"), + "url": section.get("url"), + "pushurl": section.get("pushurl"), + # September 2023: mr.developer defaults to master, mxdev to main. + "branch": section.get("branch", "main"), + "path": section.get("target"), + "egg": section.get("install-mode", "") != "skip", + } + return cls(**options) + + def __repr__(self): + return f"" + + def to_section(self): + contents = [f"[{self.name}]"] + # { 'branch': '6.0', 'path': 'extra/documentation', 'egg': False} + if self.protocol != "git": + contents.append(f"protocol = {self.protocol}") + contents.append(f"url = {self.url}") + if self.pushurl: + contents.append(f"pushurl = {self.pushurl}") + if self.branch: + contents.append(f"branch = {self.branch}") + if not self.egg: + contents.append("install-mode = skip") + if self.path: + contents.append(f"target = {self.path}") + return "\n".join(contents) + + def __str__(self): + line = f"{self.protocol} {self.url}" + if self.pushurl: + line += f" pushurl={self.pushurl}" + if self.branch: + line += f" branch={self.branch}" + if self.path: + line += f" path={self.path}" + if not self.egg: + line += " egg=false" + return line + + def __eq__(self, other): + return repr(self) == repr(other) diff --git a/plone/releaser/buildout.py b/plone/releaser/buildout.py index 2b42a61..3a68674 100644 --- a/plone/releaser/buildout.py +++ b/plone/releaser/buildout.py @@ -1,4 +1,5 @@ from .base import BaseFile +from .base import Source from .utils import buildout_marker_to_pip_marker from .utils import update_contents from collections import defaultdict @@ -13,62 +14,6 @@ import re -class Source: - """Source definition for mr.developer""" - - def __init__( - self, protocol=None, url=None, pushurl=None, branch=None, path=None, egg=True - ): - # I think mxdev only supports git as protocol. - self.protocol = protocol - self.url = url - self.pushurl = pushurl - self.branch = branch - # mxdev has target (default: sources) instead of path (default: src). - self.path = path - # egg=True: mxdev install-mode="direct" - # egg=False: mxdev install-mode="skip" - self.egg = egg - - @classmethod - def create_from_string(cls, source_string): - line_options = source_string.split() - protocol = line_options.pop(0) - url = line_options.pop(0) - # September 2023: mr.developer defaults to master, mxdev to main. - options = {"protocol": protocol, "url": url, "branch": "master"} - - # The rest of the line options are key/value pairs. - for param in line_options: - if param is not None: - key, value = param.split("=") - if key == "egg": - if value.lower() in ("true", "yes", "on"): - value = True - elif value.lower() in ("false", "no", "off"): - value = False - options[key] = value - return cls(**options) - - def __repr__(self): - return f"" - - def __str__(self): - line = f"{self.protocol} {self.url}" - if self.pushurl: - line += f" pushurl={self.pushurl}" - if self.branch: - line += f" branch={self.branch}" - if self.path: - line += f" path={self.path}" - if not self.egg: - line += " egg=false" - return line - - def __eq__(self, other): - return repr(self) == repr(other) - - class BaseBuildoutFile(BaseFile): def __init__(self, file_location, with_markers=False, read_extends=False): self.file_location = file_location @@ -316,7 +261,7 @@ def pins_to_pip(self): new_data[package] = new_version return new_data - def to_constraints(self, constraints_path): + def to_pip(self, constraints_path): """Overwrite constraints file with our data. The strategy is: @@ -351,7 +296,7 @@ def data(self): sources_dict = OrderedDict() # I don't think we need to support [sources:marker]. for name, value in self.config["sources"].items(): - source = Source.create_from_string(value) + source = Source.create_from_string(name, value) sources_dict[name] = source return sources_dict @@ -360,7 +305,7 @@ def raw_data(self): sources_dict = OrderedDict() # I don't think we need to support [sources:marker]. for name, value in self.raw_config["sources"].items(): - source = Source.create_from_string(value) + source = Source.create_from_string(name, value) sources_dict[name] = source return sources_dict @@ -396,6 +341,38 @@ def rewrite(self): new_contents = "\n".join(contents) self.path.write_text(new_contents) + def to_pip(self, pip_path): + """Overwrite mxdev/pip sources file with our data. + + The strategy is: + + 1. Translate our data to mxdev sources data. + 2. Ask the msdev sources file to rewrite itself. + """ + # Import here to avoid circular imports. + from plone.releaser.pip import MxSourcesFile + + sources = MxSourcesFile(pip_path) + # Create or empty the sources file. + sources.path.write_text("") + + # Translate our data to pip. + sources.data = self.raw_data + sources.settings = {"docs-directory": "documentation"} + if "remotes" in self.config: + remotes = self.config["remotes"] + for key, value in remotes.items(): + sources.settings[key] = value + for source in sources.data.values(): + source.url = source.url.replace("{remotes:", "{settings:") + if source.pushurl: + source.pushurl = source.pushurl.replace("{remotes:", "{settings:") + if source.path: + source.path = source.path.replace("{buildout:", "{settings:") + + # Rewrite the file. + sources.rewrite() + class CheckoutsFile(BaseBuildoutFile): @property @@ -454,6 +431,30 @@ def rewrite(self): new_contents = "\n".join(contents) self.path.write_text(new_contents) + def to_pip(self, pip_path): + """Overwrite mxdev/pip checkouts file with our data. + + The strategy is: + + 1. Translate our data to mxdev checkouts data. + 2. Ask the msdev checkouts file to rewrite itself. + """ + # Import here to avoid circular imports. + from plone.releaser.pip import MxCheckoutsFile + + checkouts = MxCheckoutsFile(pip_path) + # Create or empty the checkouts file. + checkouts.path.write_text("") + + # Translate our data to pip. + # XXX does not do anything + checkouts.data = self.data + # This is the only setting that makes sense for Plone coredev: + checkouts.settings = {"default-use": "false"} + + # Rewrite the file. + checkouts.rewrite() + class Buildout: def __init__( diff --git a/plone/releaser/manage.py b/plone/releaser/manage.py index 6a34519..b235d28 100644 --- a/plone/releaser/manage.py +++ b/plone/releaser/manage.py @@ -1,6 +1,7 @@ from argh import arg from argh import ArghParser from argh.decorators import named +from pathlib import Path from plone.releaser import ACTION_BATCH from plone.releaser import ACTION_INTERACTIVE from plone.releaser import ACTION_REPORT @@ -8,10 +9,11 @@ from plone.releaser import THIRD_PARTY_PACKAGES from plone.releaser.buildout import Buildout from plone.releaser.buildout import CheckoutsFile +from plone.releaser.buildout import SourcesFile from plone.releaser.buildout import VersionsFile from plone.releaser.package import Package from plone.releaser.pip import ConstraintsFile -from plone.releaser.pip import IniFile +from plone.releaser.pip import MxCheckoutsFile from progress.bar import Bar import glob @@ -143,7 +145,7 @@ def _get_checkouts(path=None): paths = glob.glob("mxdev.ini") + glob.glob("checkouts.cfg") for path in paths: if path.endswith(".ini"): - checkouts = IniFile(path) + checkouts = MxCheckoutsFile(path) else: checkouts = CheckoutsFile(path) yield checkouts @@ -268,33 +270,34 @@ def set_package_version(package_name, new_version, *, path=None): constraints.set(package_name, new_version) +def _get_paths(path, patterns): + paths = [] + if path: + if not isinstance(path, Path): + path = Path(path) + if path.is_dir(): + for pat in patterns: + paths.extend(glob.glob(str(path / pat))) + else: + paths = [path] + else: + for pat in patterns: + paths.extend(glob.glob(pat)) + all_paths = [] + for path in paths: + if not isinstance(path, Path): + path = Path(path) + all_paths.append(path) + return all_paths + + def versions2constraints(*, path=None): """Take a Buildout versions file and create a pip constraints file out of it. + If a path is given, we handle only that file. If no path is given, we use versions*.cfg. - - Notes: - * This does not handle 'extends' yet. - * This does not handle [versions:pythonX] yet. - - We could parse the file with Buildout. This incorporates the 'extends', - but you lose versions information for other Python versions. - - We could pass an option simple/full. - Maybe if a path is passed, we handle only that file in simple mode. - Without path, we grab versions.cfg and check 'extends' and other versions. - - 'extends = versions-extra.cfg' could be transformed to '-c constraints-extra.txt' - - I think I need some more options in VersionsFile first: - - what to do with extends - - what to do with [versions:*] - - whether to turn it into a single constraints file. """ - if path: - paths = [path] - else: - paths = glob.glob("versions*.cfg") + paths = _get_paths(path, ["versions*.cfg"]) for path in paths: versions = VersionsFile(path, with_markers=True) # Create path to constraints*.txt instead of versions*.cfg. @@ -302,7 +305,40 @@ def versions2constraints(*, path=None): filename = str(filepath)[len(str(filepath.parent)) + 1 :] filename = filename.replace("versions", "constraints").replace(".cfg", ".txt") constraints_path = filepath.parent / filename - versions.to_constraints(constraints_path) + versions.to_pip(constraints_path) + + +def buildout2pip(*, path=None): + """Take a Buildout file and create a pip/mxdev file out of it. + + If a path is given, we handle only that file, guessing whether it is a file + with versions or sources or checkouts. + If no path is given, we use versions*.cfg, sources*.cfg and checkouts*.cfg. + """ + paths = _get_paths(path, ["versions*.cfg", "sources*.cfg", "checkouts*.cfg"]) + for path in paths: + if path.name.startswith("versions"): + buildout_file = VersionsFile(path, with_markers=True) + elif path.name.startswith("sources"): + buildout_file = SourcesFile(path) + elif path.name.startswith("checkouts"): + buildout_file = CheckoutsFile(path) + # Create path to constraints*.txt instead of versions*.cfg, etc. + filepath = buildout_file.path + filename = str(filepath)[len(str(filepath.parent)) + 1 :] + filename = filename.replace("versions", "constraints") + if "checkouts" in filename or "sources" in filename: + filename = ( + filename.replace("checkouts", "mxcheckouts") + .replace("sources", "mxsources") + .replace(".cfg", ".ini") + ) + else: + filename = filename.replace(".cfg", ".txt") + pip_path = filepath.parent / filename + if not pip_path.exists(): + pip_path.write_text("") + buildout_file.to_pip(pip_path) class Manage: @@ -322,6 +358,7 @@ def __call__(self, **kwargs): get_package_version, jenkins_report, versions2constraints, + buildout2pip, ] ) parser.dispatch() diff --git a/plone/releaser/pip.py b/plone/releaser/pip.py index 67baad0..d791fa9 100644 --- a/plone/releaser/pip.py +++ b/plone/releaser/pip.py @@ -1,6 +1,8 @@ from .base import BaseFile +from .base import Source from .utils import update_contents from collections import defaultdict +from collections import OrderedDict from configparser import ConfigParser from functools import cached_property @@ -135,9 +137,64 @@ def rewrite(self): self.path.write_text(new_contents) -class IniFile(BaseFile): +class MxSourcesFile(BaseFile): """Ini file for mxdev. + What we want to do here is similar to what we have in buildout.py + in the SourcesFile. + """ + + def __init__(self, file_location): + super().__init__(file_location) + self.config = ConfigParser( + default_section="settings", + ) + # mxdev itself calls ConfigParser with extra option + # interpolation=ExtendedInterpolation(). + # This turns a line like 'url = ${settings:plone}/package.git' + # into 'url = https://github.com/plone/package.git'. + # In our case we very much want the original line, + # especially when we do a rewrite of the file. + with self.path.open() as f: + self.config.read_file(f) + + @cached_property + def data(self): + sources_dict = OrderedDict() + # I don't think we need to support [sources:marker]. + for package in self.config.sections(): + section = self.config[package] + sources_dict[package] = Source.create_from_section(section) + return sources_dict + + @cached_property + def settings(self): + return self.config["settings"] + + def __setitem__(self, package_name, enabled=True): + raise NotImplementedError + + def rewrite(self): + """Rewrite the file based on the parsed data. + + This will lose comments, and may change the order. + """ + contents = ["[settings]"] + for key, value in self.settings.items(): + contents.append(f"{key} = {value}") + + for package in self: + contents.append("") + contents.append(self[package].to_section()) + + contents.append("") + new_contents = "\n".join(contents) + self.path.write_text(new_contents) + + +class MxCheckoutsFile(BaseFile): + """Checkouts file for mxdev. + What we want to do here is similar to what we have in buildout.py in the CheckoutsFile: remove a package from auto-checkouts. For mxdev: set 'use = false'. @@ -159,7 +216,7 @@ def __init__(self, file_location): self.config.read_file(f) self.default_use = to_bool(self.config["settings"].get("default-use", True)) - @property + @cached_property def data(self): checkouts = {} for package in self.config.sections(): @@ -168,7 +225,7 @@ def data(self): checkouts[package] = True return checkouts - @property + @cached_property def sections(self): # If we want to use a package, we must first know that it exists. sections = {} @@ -176,11 +233,28 @@ def sections(self): sections[package] = True return sections + @cached_property + def settings(self): + return self.config["settings"] + @property def lowerkeys_section(self): # Map from lower case key to actual key in the sections. return {key.lower(): key for key in self.sections} + def append_package(self, package_name, enabled=True): + """Append a package to the checkouts. + + The caller should have made sure this package is currently + not in the file. + """ + contents = self.path.read_text() + if not contents.endswith("\n"): + contents += "\n" + use = "true" if enabled else "false" + contents += f"\n[{package_name}]\nuse = {use}\n" + self.path.write_text(contents) + def __setitem__(self, package_name, enabled=True): """Enable or disable a checkout. @@ -190,17 +264,15 @@ def __setitem__(self, package_name, enabled=True): But let's support the other way around as well: when default-use is true, we set 'use = false'. - - Note that in our Buildout setup, we have sources.cfg separately. - In mxdev.ini the source definition and 'use = false/true' is combined. - So if the package we want to enable is not defined, meaning it has no - section, then we should fail loudly. """ stored_package_name = self.lowerkeys_section.get(package_name.lower()) if not stored_package_name: - raise KeyError( - f"{self.file_location}: There is no definition for {package_name}" - ) + # Package is not known to us. + if self.default_use == enabled: + # The wanted state is the default state, so do nothing. + return + self.append_package(package_name, enabled=enabled) + return package_name = stored_package_name if package_name in self: use = to_bool(self.config[package_name].get("use", self.default_use)) @@ -263,19 +335,15 @@ def rewrite(self): """Rewrite the file based on the parsed data. This will lose comments, and may change the order. - TODO Can we trust self.config? It won't get updated if we change any data - after reading. """ contents = ["[settings]"] - for key, value in self.config["settings"].items(): + for key, value in self.settings.items(): contents.append(f"{key} = {value}") - for package in self.sections: + for package in self.data: contents.append("") contents.append(f"[{package}]") - for key, value in self.config[package].items(): - if self.config["settings"].get(key) != value: - contents.append(f"{key} = {value}") + contents.append("use = true") contents.append("") new_contents = "\n".join(contents) diff --git a/plone/releaser/release.py b/plone/releaser/release.py index 2a92c2b..21c0715 100644 --- a/plone/releaser/release.py +++ b/plone/releaser/release.py @@ -3,7 +3,7 @@ from plone.releaser.buildout import SourcesFile from plone.releaser.buildout import VersionsFile from plone.releaser.pip import ConstraintsFile -from plone.releaser.pip import IniFile +from plone.releaser.pip import MxSourcesFile from plone.releaser.pypi import can_user_release_package_to_pypi from zest.releaser import pypi from zest.releaser.utils import ask @@ -311,5 +311,5 @@ def remove_from_checkouts(package_name): checkouts.remove(package_name) checkouts_file = coredev_dir / "mxdev.ini" if checkouts_file.exists(): - checkouts = IniFile(checkouts_file) + checkouts = MxSourcesFile(checkouts_file) checkouts.remove(package_name) diff --git a/plone/releaser/tests/input/mxcheckouts.ini b/plone/releaser/tests/input/mxcheckouts.ini new file mode 100644 index 0000000..c7fbf35 --- /dev/null +++ b/plone/releaser/tests/input/mxcheckouts.ini @@ -0,0 +1,8 @@ +[settings] +default-use = false + +[package] +use = true + +[CamelCase] +use = true diff --git a/plone/releaser/tests/input/mxdev.ini b/plone/releaser/tests/input/mxsources.ini similarity index 68% rename from plone/releaser/tests/input/mxdev.ini rename to plone/releaser/tests/input/mxsources.ini index a0c15fd..f0e9dfd 100644 --- a/plone/releaser/tests/input/mxdev.ini +++ b/plone/releaser/tests/input/mxsources.ini @@ -1,16 +1,13 @@ [settings] requirements-in = requirements.txt requirements-out = requirements-mxdev.txt -contraints-out = constraints-mxdev.txt -# The packages are defined, but not used by default. -default-use = false +constraints-out = constraints-mxdev.txt # custom variables plone = https://github.com/plone [package] url = ${settings:plone}/package.git branch = main -use = true [unused] url = ${settings:plone}/package.git @@ -19,4 +16,9 @@ branch = main [CamelCase] url = ${settings:plone}/CamelCase.git branch = main -use = true + +[docs] +url = ${setting:plone}/documentation.git +branch = 6.0 +install-mode = skip +target = extra/documentation diff --git a/plone/releaser/tests/test_buildout.py b/plone/releaser/tests/test_buildout.py index 95e28a2..977c33e 100644 --- a/plone/releaser/tests/test_buildout.py +++ b/plone/releaser/tests/test_buildout.py @@ -1,5 +1,4 @@ from plone.releaser.buildout import CheckoutsFile -from plone.releaser.buildout import Source from plone.releaser.buildout import SourcesFile from plone.releaser.buildout import VersionsFile @@ -101,49 +100,6 @@ def test_checkouts_file_rewrite(tmp_path): ) -def test_source_standard(): - src = Source.create_from_string( - "git https://github.com/plone/Plone.git pushurl=git@github.com:plone/Plone.git branch=6.0.x" - ) - assert src.protocol == "git" - assert src.url == "https://github.com/plone/Plone.git" - assert src.pushurl == "git@github.com:plone/Plone.git" - assert src.branch == "6.0.x" - assert src.egg is True - assert src.path is None - - -def test_source_not_enough_parameters(): - with pytest.raises(IndexError): - Source.create_from_string("") - with pytest.raises(IndexError): - Source.create_from_string("git") - - -def test_source_just_enough_parameters(): - # protocol and url are enough - src = Source.create_from_string("git https://github.com/plone/Plone.git") - assert src.protocol == "git" - assert src.url == "https://github.com/plone/Plone.git" - assert src.pushurl is None - assert src.branch == "master" - assert src.egg is True - assert src.path is None - - -def test_source_docs(): - # Plone has a docs source with some extra options. - src = Source.create_from_string( - "git https://github.com/plone/documentation.git pushurl=git@github.com:plone/documentation.git egg=false branch=6.0 path=docs" - ) - assert src.protocol == "git" - assert src.url == "https://github.com/plone/documentation.git" - assert src.pushurl == "git@github.com:plone/documentation.git" - assert src.branch == "6.0" - assert src.egg is False - assert src.path == "docs" - - def test_sources_file_data(): sf = SourcesFile(SOURCES_FILE) assert sorted(sf.data.keys()) == ["Plone", "docs", "plone.alterego", "plone.base"] diff --git a/plone/releaser/tests/test_buildout2pip.py b/plone/releaser/tests/test_buildout2pip.py new file mode 100644 index 0000000..4d8b7ab --- /dev/null +++ b/plone/releaser/tests/test_buildout2pip.py @@ -0,0 +1,115 @@ +from plone.releaser.manage import buildout2pip +from plone.releaser.pip import ConstraintsFile + +import pathlib +import shutil + + +TESTS_DIR = pathlib.Path(__file__).parent +INPUT_DIR = TESTS_DIR / "input" +VERSIONS_FILE = INPUT_DIR / "versions.cfg" +SOURCES_FILE = INPUT_DIR / "sources.cfg" +CHECKOUTS_FILE = INPUT_DIR / "checkouts.cfg" + + +def test_buildout2pip_one_path(tmp_path): + copy_path = tmp_path / "versions.cfg" + constraints_file = tmp_path / "constraints.txt" + shutil.copyfile(VERSIONS_FILE, copy_path) + assert not constraints_file.exists() + buildout2pip(path=copy_path) + assert constraints_file.exists() + cf = ConstraintsFile(constraints_file, with_markers=True) + assert cf.data == { + "CamelCase": "1.0", + "UPPERCASE": "1.0", + "annotated": "1.0", + "duplicate": "1.0", + "lowercase": "1.0", + "onepython": {'python_version == "3.12"': "2.1"}, + "package": "1.0", + "pyspecific": {"": "1.0", 'python_version == "3.12"': "2.0"}, + } + assert ( + constraints_file.read_text() + == """-c https://zopefoundation.github.io/Zope/releases/5.8.3/constraints.txt +annotated==1.0 +CamelCase==1.0 +duplicate==1.0 +lowercase==1.0 +package==1.0 +pyspecific==1.0 +pyspecific==2.0; python_version == "3.12" +UPPERCASE==1.0 +onepython==2.1; python_version == "3.12" +""" + ) + + +def test_buildout2pip_all(tmp_path): + versions_path = tmp_path / "versions.cfg" + constraints_file = tmp_path / "constraints.txt" + shutil.copyfile(VERSIONS_FILE, versions_path) + sources_path = tmp_path / "sources.cfg" + mxsources_file = tmp_path / "mxsources.ini" + shutil.copyfile(SOURCES_FILE, sources_path) + checkouts_path = tmp_path / "checkouts.cfg" + mxcheckouts_file = tmp_path / "mxcheckouts.ini" + shutil.copyfile(CHECKOUTS_FILE, checkouts_path) + buildout2pip(path=tmp_path) + assert constraints_file.exists() + assert mxsources_file.exists() + assert mxcheckouts_file.exists() + assert ( + constraints_file.read_text() + == """-c https://zopefoundation.github.io/Zope/releases/5.8.3/constraints.txt +annotated==1.0 +CamelCase==1.0 +duplicate==1.0 +lowercase==1.0 +package==1.0 +pyspecific==1.0 +pyspecific==2.0; python_version == "3.12" +UPPERCASE==1.0 +onepython==2.1; python_version == "3.12" +""" + ) + assert ( + mxsources_file.read_text() + == """[settings] +docs-directory = documentation +plone = https://github.com/plone +plone_push = git@github.com:plone + +[docs] +url = ${settings:plone}/documentation.git +branch = 6.0 +install-mode = skip +target = ${settings:docs-directory} + +[Plone] +url = ${settings:plone}/Plone.git +pushurl = ${settings:plone_push}/Plone.git +branch = 6.0.x + +[plone.alterego] +url = ${settings:plone}/plone.alterego.git +branch = master + +[plone.base] +url = ${settings:plone}/plone.base.git +branch = main +""" + ) + assert ( + mxcheckouts_file.read_text() + == """[settings] +default-use = false + +[CamelCase] +use = true + +[package] +use = true +""" + ) diff --git a/plone/releaser/tests/test_mxdev_checkouts.py b/plone/releaser/tests/test_mxdev_checkouts.py new file mode 100644 index 0000000..f177bb0 --- /dev/null +++ b/plone/releaser/tests/test_mxdev_checkouts.py @@ -0,0 +1,118 @@ +from plone.releaser.pip import MxCheckoutsFile + +import pathlib +import pytest +import shutil + + +TESTS_DIR = pathlib.Path(__file__).parent +INPUT_DIR = TESTS_DIR / "input" +MX_CHECKOUTS_FILE = INPUT_DIR / "mxcheckouts.ini" + + +def test_mx_checkouts_file_data(): + mf = MxCheckoutsFile(MX_CHECKOUTS_FILE) + # The data used to map lower case to actual case, + # but now actual case to True. + assert mf.data == { + "CamelCase": True, + "package": True, + } + + +def test_mx_checkouts_file_contains(): + mf = MxCheckoutsFile(MX_CHECKOUTS_FILE) + assert "package" in mf + assert "unused" not in mf + # We compare case insensitively. + assert "camelcase" in mf + assert "CamelCase" in mf + assert "CAMELCASE" in mf + + +def test_mx_checkouts_file_get(): + mf = MxCheckoutsFile(MX_CHECKOUTS_FILE) + assert mf["package"] is True + assert mf.get("package") is True + assert mf["camelcase"] is True + assert mf["CAMELCASE"] is True + assert mf["CamelCase"] is True + with pytest.raises(KeyError): + mf["unused"] + assert mf.get("unused") is None + + +def test_mx_checkouts_file_add_known(tmp_path): + # When we add or remove a checkout, the file changes, so we work on a copy. + copy_path = tmp_path / "mxdev.ini" + shutil.copyfile(MX_CHECKOUTS_FILE, copy_path) + mf = MxCheckoutsFile(copy_path) + assert "unused" not in mf + mf.add("unused") + # Let's read it fresh, for good measure. + mf = MxCheckoutsFile(copy_path) + assert "unused" in mf + assert mf["unused"] is True + + +def test_mx_checkouts_file_add_unknown(tmp_path): + # It actually does not matter for us if a package is unknown in a + # mxdev sources file: we are happy to add it in a mxdev checkouts file. + copy_path = tmp_path / "mxdev.ini" + shutil.copyfile(MX_CHECKOUTS_FILE, copy_path) + mf = MxCheckoutsFile(copy_path) + assert "unknown" not in mf + mf.add("unknown") + # Let's read it fresh, for good measure. + mf = MxCheckoutsFile(copy_path) + assert "unknown" in mf + assert mf["unknown"] is True + + +def test_mx_checkouts_file_remove(tmp_path): + copy_path = tmp_path / "mxdev.ini" + shutil.copyfile(MX_CHECKOUTS_FILE, copy_path) + mf = MxCheckoutsFile(copy_path) + assert "package" in mf + mf.remove("package") + # Let's read it fresh, for good measure. + mf = MxCheckoutsFile(copy_path) + assert "package" not in mf + assert "CAMELCASE" in mf + mf.remove("CAMELCASE") + mf = MxCheckoutsFile(copy_path) + assert "CAMELCASE" not in mf + assert "CamelCase" not in mf + assert "camelcase" not in mf + # Check that we can re-enable a package: + # editing should not remove the entire section. + mf.add("package") + mf = MxCheckoutsFile(copy_path) + assert "package" in mf + # This should work for the last section as well. + mf.add("CamelCase") + mf = MxCheckoutsFile(copy_path) + assert "CamelCase" in mf + + +def test_mx_checkouts_file_rewrite(tmp_path): + copy_path = tmp_path / "mxsources.ini" + shutil.copyfile(MX_CHECKOUTS_FILE, copy_path) + mf = MxCheckoutsFile(copy_path) + mf.rewrite() + # Read it fresh and compare + mf2 = MxCheckoutsFile(copy_path) + assert mf.data == mf2.data + # Check the entire text. + assert ( + copy_path.read_text() + == """[settings] +default-use = false + +[package] +use = true + +[CamelCase] +use = true +""" + ) diff --git a/plone/releaser/tests/test_mxdev_sources.py b/plone/releaser/tests/test_mxdev_sources.py new file mode 100644 index 0000000..18db59b --- /dev/null +++ b/plone/releaser/tests/test_mxdev_sources.py @@ -0,0 +1,97 @@ +from plone.releaser.base import Source +from plone.releaser.pip import MxSourcesFile + +import pathlib +import pytest +import shutil + + +TESTS_DIR = pathlib.Path(__file__).parent +INPUT_DIR = TESTS_DIR / "input" +MX_SOURCES_FILE = INPUT_DIR / "mxsources.ini" + + +def test_mx_sources_file_data(): + mf = MxSourcesFile(MX_SOURCES_FILE) + assert list(mf.data.keys()) == ["package", "unused", "CamelCase", "docs"] + for key, value in mf.data.items(): + assert isinstance(value, Source) + + +def test_mx_sources_file_contains(): + mf = MxSourcesFile(MX_SOURCES_FILE) + assert "package" in mf + assert "unused" in mf + # We compare case insensitively. + assert "camelcase" in mf + assert "CamelCase" in mf + assert "CAMELCASE" in mf + + +def test_mx_sources_file_get(): + mf = MxSourcesFile(MX_SOURCES_FILE) + package = mf["package"] + assert package + assert mf.get("package") == package + assert isinstance(package, Source) + assert package.protocol == "git" + assert package.url == "${settings:plone}/package.git" + assert package.branch == "main" + assert package.pushurl is None + assert package.path is None + assert package.egg is True + lowercase = mf["camelcase"] + uppercase = mf["CAMELCASE"] + camelcase = mf["CamelCase"] + assert lowercase + assert isinstance(lowercase, Source) + assert uppercase + assert camelcase + assert lowercase == uppercase + assert lowercase == camelcase + # The 'unused' package is not used (checked out), but we do not know that. + assert isinstance(mf.get("unused"), Source) + with pytest.raises(KeyError): + mf["no-such-package"] + docs = mf["docs"] + assert docs.branch == "6.0" + assert docs.egg is False + assert docs.path == "extra/documentation" + + +def test_mx_sources_file_rewrite(tmp_path): + copy_path = tmp_path / "mxdev.ini" + shutil.copyfile(MX_SOURCES_FILE, copy_path) + mf = MxSourcesFile(copy_path) + mf.rewrite() + # Read it fresh and compare + mf2 = MxSourcesFile(copy_path) + assert mf.data == mf2.data + # Check the entire text. + assert ( + copy_path.read_text() + == """[settings] +requirements-in = requirements.txt +requirements-out = requirements-mxdev.txt +constraints-out = constraints-mxdev.txt +plone = https://github.com/plone + +[package] +url = ${settings:plone}/package.git +branch = main + +[unused] +url = ${settings:plone}/package.git +branch = main + +[CamelCase] +url = ${settings:plone}/CamelCase.git +branch = main + +[docs] +url = ${setting:plone}/documentation.git +branch = 6.0 +install-mode = skip +target = extra/documentation +""" + ) diff --git a/plone/releaser/tests/test_pip.py b/plone/releaser/tests/test_pip.py index 7386364..0c9cff4 100644 --- a/plone/releaser/tests/test_pip.py +++ b/plone/releaser/tests/test_pip.py @@ -1,5 +1,4 @@ from plone.releaser.pip import ConstraintsFile -from plone.releaser.pip import IniFile import pathlib import pytest @@ -12,124 +11,6 @@ CONSTRAINTS_FILE2 = INPUT_DIR / "constraints2.txt" CONSTRAINTS_FILE3 = INPUT_DIR / "constraints3.txt" CONSTRAINTS_FILE4 = INPUT_DIR / "constraints4.txt" -MXDEV_FILE = INPUT_DIR / "mxdev.ini" - - -def test_mxdev_file_data(): - mf = IniFile(MXDEV_FILE) - # The data used to map lower case to actual case, - # but now actual case to True. - assert mf.data == { - "CamelCase": True, - "package": True, - } - - -def test_mxdev_file_contains(): - mf = IniFile(MXDEV_FILE) - assert "package" in mf - assert "unused" not in mf - # We compare case insensitively. - assert "camelcase" in mf - assert "CamelCase" in mf - assert "CAMELCASE" in mf - - -def test_mxdev_file_get(): - mf = IniFile(MXDEV_FILE) - assert mf["package"] is True - assert mf.get("package") is True - assert mf["camelcase"] is True - assert mf["CAMELCASE"] is True - assert mf["CamelCase"] is True - with pytest.raises(KeyError): - mf["unused"] - assert mf.get("unused") is None - - -def test_mxdev_file_add_known(tmp_path): - # When we add or remove a checkout, the file changes, so we work on a copy. - copy_path = tmp_path / "mxdev.ini" - shutil.copyfile(MXDEV_FILE, copy_path) - mf = IniFile(copy_path) - assert "unused" not in mf - mf.add("unused") - # Let's read it fresh, for good measure. - mf = IniFile(copy_path) - assert "unused" in mf - assert mf["unused"] is True - - -def test_mxdev_file_add_unknown(tmp_path): - # We cannot edit mxdev.ini to use a package when it is not defined. - copy_path = tmp_path / "mxdev.ini" - shutil.copyfile(MXDEV_FILE, copy_path) - mf = IniFile(copy_path) - assert "unknown" not in mf - with pytest.raises(KeyError): - mf.add("unknown") - - -def test_mxdev_file_remove(tmp_path): - copy_path = tmp_path / "mxdev.ini" - shutil.copyfile(MXDEV_FILE, copy_path) - mf = IniFile(copy_path) - assert "package" in mf - mf.remove("package") - # Let's read it fresh, for good measure. - mf = IniFile(copy_path) - assert "package" not in mf - assert "CAMELCASE" in mf - mf.remove("CAMELCASE") - mf = IniFile(copy_path) - assert "CAMELCASE" not in mf - assert "CamelCase" not in mf - assert "camelcase" not in mf - # Check that we can re-enable a package: - # editing should not remove the entire section. - mf.add("package") - mf = IniFile(copy_path) - assert "package" in mf - # This should work for the last section as well. - mf.add("CamelCase") - mf = IniFile(copy_path) - assert "CamelCase" in mf - - -def test_mxdev_file_rewrite(tmp_path): - copy_path = tmp_path / "mxdev.ini" - shutil.copyfile(MXDEV_FILE, copy_path) - mf = IniFile(copy_path) - mf.rewrite() - # Read it fresh and compare - mf2 = IniFile(copy_path) - assert mf.data == mf2.data - # Check the entire text. Note that packages are alphabetically sorted. - # Currently we get the original case, but we may change this to lowercase. - assert ( - copy_path.read_text() - == """[settings] -requirements-in = requirements.txt -requirements-out = requirements-mxdev.txt -contraints-out = constraints-mxdev.txt -default-use = false -plone = https://github.com/plone - -[package] -url = ${settings:plone}/package.git -branch = main -use = true - -[unused] -url = ${settings:plone}/package.git -branch = main - -[CamelCase] -url = ${settings:plone}/CamelCase.git -branch = main -use = true -""" - ) def test_constraints_file_constraints(): diff --git a/plone/releaser/tests/test_source.py b/plone/releaser/tests/test_source.py new file mode 100644 index 0000000..6221674 --- /dev/null +++ b/plone/releaser/tests/test_source.py @@ -0,0 +1,84 @@ +from configparser import ConfigParser +from plone.releaser.buildout import Source + +import pytest + + +def test_source_standard(): + src = Source.create_from_string( + "Plone", + "git https://github.com/plone/Plone.git pushurl=git@github.com:plone/Plone.git branch=6.0.x", + ) + assert src.name == "Plone" + assert src.protocol == "git" + assert src.url == "https://github.com/plone/Plone.git" + assert src.pushurl == "git@github.com:plone/Plone.git" + assert src.branch == "6.0.x" + assert src.egg is True + assert src.path is None + + +def test_source_not_enough_parameters(): + with pytest.raises(IndexError): + Source.create_from_string("package", "") + with pytest.raises(IndexError): + Source.create_from_string("package", "git") + + +def test_source_just_enough_parameters(): + # protocol and url are enough + src = Source.create_from_string("Plone", "git https://github.com/plone/Plone.git") + assert src.name == "Plone" + assert src.protocol == "git" + assert src.url == "https://github.com/plone/Plone.git" + assert src.pushurl is None + assert src.branch == "master" + assert src.egg is True + assert src.path is None + + +def test_source_docs(): + # Plone has a docs source with some extra options. + src = Source.create_from_string( + "docs", + "git https://github.com/plone/documentation.git pushurl=git@github.com:plone/documentation.git egg=false branch=6.0 path=docs", + ) + assert src.name == "docs" + assert src.protocol == "git" + assert src.url == "https://github.com/plone/documentation.git" + assert src.pushurl == "git@github.com:plone/documentation.git" + assert src.branch == "6.0" + assert src.egg is False + assert src.path == "docs" + + +def test_source_from_section(): + config = ConfigParser() + config.read_string("[Plone]\nurl = blah") + src = Source.create_from_section(config["Plone"]) + assert src.name == "Plone" + assert src.url == "blah" + assert not src.pushurl + assert src.branch == "main" + assert src.egg + assert not src.path + + config.read_string( + "\n".join( + [ + "[package]", + "url = hop", + "pushurl = other", + "branch = 1.x", + "install-mode=skip", + "target = /some/path", + ] + ) + ) + src = Source.create_from_section(config["package"]) + assert src.name == "package" + assert src.url == "hop" + assert src.pushurl == "other" + assert src.branch == "1.x" + assert not src.egg + assert src.path == "/some/path" diff --git a/plone/releaser/tests/test_versions2constraints.py b/plone/releaser/tests/test_versions2constraints.py index 42dc77c..2cf96a6 100644 --- a/plone/releaser/tests/test_versions2constraints.py +++ b/plone/releaser/tests/test_versions2constraints.py @@ -2,7 +2,6 @@ from plone.releaser.pip import ConstraintsFile import pathlib -import pytest import shutil @@ -14,8 +13,7 @@ VERSIONS_FILE4 = INPUT_DIR / "versions4.cfg" -@pytest.mark.current -def test_versions2constraints(tmp_path): +def test_versions2constraints_one_path(tmp_path): copy_path = tmp_path / "versions.cfg" constraints_file = tmp_path / "constraints.txt" shutil.copyfile(VERSIONS_FILE, copy_path) @@ -23,7 +21,16 @@ def test_versions2constraints(tmp_path): versions2constraints(path=copy_path) assert constraints_file.exists() cf = ConstraintsFile(constraints_file, with_markers=True) - print(cf.data) + assert cf.data == { + "CamelCase": "1.0", + "UPPERCASE": "1.0", + "annotated": "1.0", + "duplicate": "1.0", + "lowercase": "1.0", + "onepython": {'python_version == "3.12"': "2.1"}, + "package": "1.0", + "pyspecific": {"": "1.0", 'python_version == "3.12"': "2.0"}, + } assert ( constraints_file.read_text() == """-c https://zopefoundation.github.io/Zope/releases/5.8.3/constraints.txt @@ -38,3 +45,87 @@ def test_versions2constraints(tmp_path): onepython==2.1; python_version == "3.12" """ ) + + +def test_versions2constraints_all(tmp_path): + versions_path = tmp_path / "versions.cfg" + versions2_path = tmp_path / "versions2.cfg" + versions3_path = tmp_path / "versions3.cfg" + versions4_path = tmp_path / "versions4.cfg" + constraints_file = tmp_path / "constraints.txt" + constraints2_file = tmp_path / "constraints2.txt" + constraints3_file = tmp_path / "constraints3.txt" + constraints4_file = tmp_path / "constraints4.txt" + shutil.copyfile(VERSIONS_FILE, versions_path) + shutil.copyfile(VERSIONS_FILE2, versions2_path) + shutil.copyfile(VERSIONS_FILE3, versions3_path) + shutil.copyfile(VERSIONS_FILE4, versions4_path) + assert not constraints_file.exists() + assert not constraints2_file.exists() + assert not constraints3_file.exists() + assert not constraints4_file.exists() + versions2constraints(path=tmp_path) + assert constraints_file.exists() + assert constraints2_file.exists() + assert constraints3_file.exists() + assert constraints4_file.exists() + cf = ConstraintsFile(constraints_file, with_markers=True) + assert cf.data == { + "CamelCase": "1.0", + "UPPERCASE": "1.0", + "annotated": "1.0", + "duplicate": "1.0", + "lowercase": "1.0", + "onepython": {'python_version == "3.12"': "2.1"}, + "package": "1.0", + "pyspecific": {"": "1.0", 'python_version == "3.12"': "2.0"}, + } + assert ( + constraints_file.read_text() + == """-c https://zopefoundation.github.io/Zope/releases/5.8.3/constraints.txt +annotated==1.0 +CamelCase==1.0 +duplicate==1.0 +lowercase==1.0 +package==1.0 +pyspecific==1.0 +pyspecific==2.0; python_version == "3.12" +UPPERCASE==1.0 +onepython==2.1; python_version == "3.12" +""" + ) + cf2 = ConstraintsFile(constraints2_file, with_markers=True) + assert cf2.data == { + "one": "1.1", + "three": {'python_version == "3.12"': "3.2"}, + "two": "2.0", + } + assert ( + constraints2_file.read_text() + == """-c constraints3.txt +one==1.1 +two==2.0 +three==3.2; python_version == "3.12" +""" + ) + cf3 = ConstraintsFile(constraints3_file, with_markers=True) + assert cf3.data == { + "one": "1.0", + "three": {"": "3.0", 'python_version == "3.12"': "3.1"}, + } + assert ( + constraints3_file.read_text() + == """-c constraints4.txt +one==1.0 +three==3.0 +three==3.1; python_version == "3.12" +""" + ) + cf4 = ConstraintsFile(constraints4_file, with_markers=True) + assert cf4.data == {"four": "4.0", "five": {'platform_system == "Darwin"': "5.0"}} + assert ( + constraints4_file.read_text() + == """four==4.0 +five==5.0; platform_system == "Darwin" +""" + ) diff --git a/setup.py b/setup.py index 35a725e..2c98ce3 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup -version = "2.2.3.dev0" +version = "2.3.0.dev0" long_description = "{}\n{}".format( open("README.rst").read(), open("CHANGES.rst").read()