Merge pull request #72 from plone/maurits-pip-sources

Add buildout2pip command

README.rst

@@ -2,22 +2,59 @@ Introduction
 ============
 
 Tools to make managing Plone core releases easier.
+It is a wrapper around ``zest.releaser``, plus it adds some commands.
+
+WARNING: this package is only meant for development of core Plone.
+It may have useful bits for others, but we may drop features and edge cases at any time if it is no longer useful for Plone.
+
 
 Installation
 ------------
 
-To install plone.releaser add it to your buildout::
+Do ``pip install plone.releaser`` or add it to your buildout::
 
   [release]
   recipe = zc.recipe.egg
-  eggs =  plone.releaser
+  eggs = plone.releaser
+
+
+Main usage: release a package
+-----------------------------
+
+In  the `Plone core development buildout <https://github.com/plone/buildout.coredev>`_ go to ``src/some.package`` and run ``../../bin/fullrelease``.
+This calls the ``fullrelease`` command from ``zest.releaser``, but with some extra hooks and packages available.
+
+One nice thing it does: look in the checkouts and sources of your ``buildout.coredev`` checkout and update them:
+remove the released package from the checkouts and update the version.
+
+If you are working on branch 6.1 of coredev, then we check if you also have branches 5.2 and 6.0 checked out.
+There we check if the branch of the released package is in the sources.
+If you make a release from package branch ``main`` and this is the branch used in the sources, then we update the checkouts and sources of this coredev branch as well.
+
+After releasing a package, you should wait a few minutes before you manually push the changes to all coredev branches.
+This gives the PyPI mirrors time to catch up so the new release is available, so Jenkins and GitHub Actions can find it.
+
 
-To make it available in buildout.coredev, run buildout with releaser.cfg::
+Main commands
+-------------
 
-  $ bin/buildout -c releaser.cfg
+Take several Buildout files and create pip/mxdev files out of them::
 
-Usage
------
+  $ bin/manage buildout2pip
+
+Take a Buildout versions file and create a pip constraints file out of it.
+
+  $ bin/manage versions2constraints
+
+Generate a changelog with changes from all packages since a certain Plone release::
+
+  $ bin/manage changelog --start=6.1.0a1
+
+
+Other commands
+--------------
+
+Some commands are not used much (by Maurits anyway) because they are less needed these days.
 
 Check PyPi access to all Plone packages for a certain user::
 
@@ -35,10 +72,6 @@ Pulls::
 
   $ bin/manage pulls
 
-Changelog::
-
-  $ bin/manage changelog
-
 Check checkout::
 
   $ bin/manage check-checkout

news/72.feature

@@ -0,0 +1,2 @@
+Add buildout2pip manage command.
+[maurits]

plone/releaser/base.py

@@ -54,3 +54,96 @@ def add(self, package_name):
 
     def remove(self, package_name):
         return self.__delitem__(package_name)
+
+
+class Source:
+    """Source definition for mr.developer or mxdev"""
+
+    def __init__(
+        self,
+        name="",
+        protocol=None,
+        url=None,
+        pushurl=None,
+        branch=None,
+        path=None,
+        egg=True,
+    ):
+        self.name = name
+        self.protocol = protocol
+        self.url = url
+        self.pushurl = pushurl
+        self.branch = branch
+        # mxdev has target (default: sources) instead of path (default: src).
+        self.path = path
+        # egg=True: mxdev install-mode="direct"
+        # egg=False: mxdev install-mode="skip"
+        self.egg = egg
+
+    @classmethod
+    def create_from_string(cls, name, source_string):
+        line_options = source_string.split()
+        protocol = line_options.pop(0)
+        url = line_options.pop(0)
+        # September 2023: mr.developer defaults to master, mxdev to main.
+        options = {"name": name, "protocol": protocol, "url": url, "branch": "master"}
+
+        # The rest of the line options are key/value pairs.
+        for param in line_options:
+            if param is not None:
+                key, value = param.split("=")
+                if key == "egg":
+                    if value.lower() in ("true", "yes", "on"):
+                        value = True
+                    elif value.lower() in ("false", "no", "off"):
+                        value = False
+                options[key] = value
+        return cls(**options)
+
+    @classmethod
+    def create_from_section(cls, section):
+        options = {
+            "name": section.name,
+            "protocol": section.get("cvs", "git"),
+            "url": section.get("url"),
+            "pushurl": section.get("pushurl"),
+            # September 2023: mr.developer defaults to master, mxdev to main.
+            "branch": section.get("branch", "main"),
+            "path": section.get("target"),
+            "egg": section.get("install-mode", "") != "skip",
+        }
+        return cls(**options)
+
+    def __repr__(self):
+        return f"<Source name={self.name} protocol={self.protocol} url={self.url} pushurl={self.pushurl} branch={self.branch} path={self.path} egg={self.egg}>"
+
+    def to_section(self):
+        contents = [f"[{self.name}]"]
+        # { 'branch': '6.0', 'path': 'extra/documentation', 'egg': False}
+        if self.protocol != "git":
+            contents.append(f"protocol = {self.protocol}")
+        contents.append(f"url = {self.url}")
+        if self.pushurl:
+            contents.append(f"pushurl = {self.pushurl}")
+        if self.branch:
+            contents.append(f"branch = {self.branch}")
+        if not self.egg:
+            contents.append("install-mode = skip")
+        if self.path:
+            contents.append(f"target = {self.path}")
+        return "\n".join(contents)
+
+    def __str__(self):
+        line = f"{self.protocol} {self.url}"
+        if self.pushurl:
+            line += f" pushurl={self.pushurl}"
+        if self.branch:
+            line += f" branch={self.branch}"
+        if self.path:
+            line += f" path={self.path}"
+        if not self.egg:
+            line += " egg=false"
+        return line
+
+    def __eq__(self, other):
+        return repr(self) == repr(other)

plone/releaser/buildout.py

@@ -1,4 +1,5 @@
 from .base import BaseFile
+from .base import Source
 from .utils import buildout_marker_to_pip_marker
 from .utils import update_contents
 from collections import defaultdict
@@ -13,62 +14,6 @@
 import re
 
 
-class Source:
-    """Source definition for mr.developer"""
-
-    def __init__(
-        self, protocol=None, url=None, pushurl=None, branch=None, path=None, egg=True
-    ):
-        # I think mxdev only supports git as protocol.
-        self.protocol = protocol
-        self.url = url
-        self.pushurl = pushurl
-        self.branch = branch
-        # mxdev has target (default: sources) instead of path (default: src).
-        self.path = path
-        # egg=True: mxdev install-mode="direct"
-        # egg=False: mxdev install-mode="skip"
-        self.egg = egg
-
-    @classmethod
-    def create_from_string(cls, source_string):
-        line_options = source_string.split()
-        protocol = line_options.pop(0)
-        url = line_options.pop(0)
-        # September 2023: mr.developer defaults to master, mxdev to main.
-        options = {"protocol": protocol, "url": url, "branch": "master"}
-
-        # The rest of the line options are key/value pairs.
-        for param in line_options:
-            if param is not None:
-                key, value = param.split("=")
-                if key == "egg":
-                    if value.lower() in ("true", "yes", "on"):
-                        value = True
-                    elif value.lower() in ("false", "no", "off"):
-                        value = False
-                options[key] = value
-        return cls(**options)
-
-    def __repr__(self):
-        return f"<Source {self.protocol} url={self.url} pushurl={self.pushurl} branch={self.branch} path={self.path} egg={self.egg}>"
-
-    def __str__(self):
-        line = f"{self.protocol} {self.url}"
-        if self.pushurl:
-            line += f" pushurl={self.pushurl}"
-        if self.branch:
-            line += f" branch={self.branch}"
-        if self.path:
-            line += f" path={self.path}"
-        if not self.egg:
-            line += " egg=false"
-        return line
-
-    def __eq__(self, other):
-        return repr(self) == repr(other)
-
-
 class BaseBuildoutFile(BaseFile):
     def __init__(self, file_location, with_markers=False, read_extends=False):
         self.file_location = file_location
@@ -316,7 +261,7 @@ def pins_to_pip(self):
             new_data[package] = new_version
         return new_data
 
-    def to_constraints(self, constraints_path):
+    def to_pip(self, constraints_path):
         """Overwrite constraints file with our data.
 
         The strategy is:
@@ -351,7 +296,7 @@ def data(self):
         sources_dict = OrderedDict()
         # I don't think we need to support [sources:marker].
         for name, value in self.config["sources"].items():
-            source = Source.create_from_string(value)
+            source = Source.create_from_string(name, value)
             sources_dict[name] = source
         return sources_dict
 
@@ -360,7 +305,7 @@ def raw_data(self):
         sources_dict = OrderedDict()
         # I don't think we need to support [sources:marker].
         for name, value in self.raw_config["sources"].items():
-            source = Source.create_from_string(value)
+            source = Source.create_from_string(name, value)
             sources_dict[name] = source
         return sources_dict
 
@@ -396,6 +341,38 @@ def rewrite(self):
         new_contents = "\n".join(contents)
         self.path.write_text(new_contents)
 
+    def to_pip(self, pip_path):
+        """Overwrite mxdev/pip sources file with our data.
+
+        The strategy is:
+
+        1. Translate our data to mxdev sources data.
+        2. Ask the msdev sources file to rewrite itself.
+        """
+        # Import here to avoid circular imports.
+        from plone.releaser.pip import MxSourcesFile
+
+        sources = MxSourcesFile(pip_path)
+        # Create or empty the sources file.
+        sources.path.write_text("")
+
+        # Translate our data to pip.
+        sources.data = self.raw_data
+        sources.settings = {"docs-directory": "documentation"}
+        if "remotes" in self.config:
+            remotes = self.config["remotes"]
+            for key, value in remotes.items():
+                sources.settings[key] = value
+            for source in sources.data.values():
+                source.url = source.url.replace("{remotes:", "{settings:")
+                if source.pushurl:
+                    source.pushurl = source.pushurl.replace("{remotes:", "{settings:")
+                if source.path:
+                    source.path = source.path.replace("{buildout:", "{settings:")
+
+        # Rewrite the file.
+        sources.rewrite()
+
 
 class CheckoutsFile(BaseBuildoutFile):
     @property
@@ -454,6 +431,30 @@ def rewrite(self):
         new_contents = "\n".join(contents)
         self.path.write_text(new_contents)
 
+    def to_pip(self, pip_path):
+        """Overwrite mxdev/pip checkouts file with our data.
+
+        The strategy is:
+
+        1. Translate our data to mxdev checkouts data.
+        2. Ask the msdev checkouts file to rewrite itself.
+        """
+        # Import here to avoid circular imports.
+        from plone.releaser.pip import MxCheckoutsFile
+
+        checkouts = MxCheckoutsFile(pip_path)
+        # Create or empty the checkouts file.
+        checkouts.path.write_text("")
+
+        # Translate our data to pip.
+        # XXX does not do anything
+        checkouts.data = self.data
+        # This is the only setting that makes sense for Plone coredev:
+        checkouts.settings = {"default-use": "false"}
+
+        # Rewrite the file.
+        checkouts.rewrite()
+
 
 class Buildout:
     def __init__(