Merge remote-tracking branch 'upstream/master' into dont_get_platform…

…_too_soon

* upstream/master: (70 commits)
  Tweak scan CLI help. (cylc#5405)
  Add support for python 3.11 (cylc#5497)
  tests/u: add parameter ids
  build(deps): bump pypa/gh-action-pypi-publish from 1.8.5 to 1.8.6
  prerequisite: remove unused interface
  Fix change log dup section.
  Fix flaky test
  Update CONTRIBUTING.md
  Tidy platforms
  modify CONTRIBUTING.md
  Bump dev version (cylc#5501)
  Prepare release 8.1.3
  Ignore off-sequence parents, for datastore. (cylc#5495)
  Avoid duplicate prerequisites from multiple recurrences. (cylc#5466)
  Update cylc/flow/prerequisite.py
  prerequsite: remove target_point_strings attribute
  tests/f: fix events/11
  swarm: convert cylc-dev Docker image to run on ubuntu:latest
  fix changelog
  Apply prerequisite changes to spawned tasks, on reload & restart (cylc#5334)
  ...

.dockerignore

@@ -9,3 +9,4 @@ __pycache__
 !cylc
 !dockerfiles/*
 !setup*
+!MANIFEST.in

.github/workflows/1_create_release_pr.yml

@@ -32,8 +32,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          # return to 3.x once compatible with 3.10+ (pyuv)
-          python-version: '3.9'
+          python-version: '3.x'
 
       - name: Create & checkout PR branch
         uses: cylc/release-actions/stage-1/checkout-pr-branch@v1

.github/workflows/2_auto_publish_release.yml

@@ -29,8 +29,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v4
       with:
-        # return to 3.x once compatible with 3.10+ (pyuv)
-        python-version: '3.9'
+        python-version: '3.x'
 
     - name: Get the version number
       uses: cylc/release-actions/stage-2/get-version-from-pr@v1
@@ -39,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.6.4
+      uses: pypa/gh-action-pypi-publish@v1.8.6
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/build.yml

@@ -18,16 +18,16 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu-latest']
-        python: ['3.7', '3.8', '3.9', '3.10']
+        python: ['3.7', '3.8', '3.9', '3.10', '3.11']
         include:
           - os: 'macos-latest'
             python: '3.7'
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
 

.github/workflows/test_fast.yml

@@ -20,7 +20,7 @@ jobs:
       fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
         include:
           - os: 'macos-latest'
             python-version: '3.7'

.github/workflows/test_functional.yml

@@ -46,9 +46,9 @@ jobs:
         # NOTE: includes must define ALL of the matrix values
         include:
           # latest python
-          - name: 'py-3.10'
+          - name: 'py-3.11'
             os: 'ubuntu-latest'
-            python-version: '3.10'
+            python-version: '3.11'
             test-base: 'tests/f'
             chunk: '1/4'
             platform: '_local_background*'

.github/workflows/test_manylinux.yml

@@ -34,7 +34,7 @@ jobs:
       matrix:
         manylinux: ['1']
         os: ['ubuntu-20.04']  # run on the oldest linux we have access to
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
       - name: Checkout

CHANGES.md

@@ -9,18 +9,55 @@ creating a new release entry be sure to copy & paste the span tag with the
 `actions:bind` attribute, which is used by a regex to find the text to be
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
+
 -------------------------------------------------------------------------------
+## __cylc-8.2.0 (<span actions:bind='release-date'>Upcoming</span>)__
+
+### Enhancements
+
+[#5405](https://github.com/cylc/cylc-flow/pull/5405) - Improve scan command
+help, and add scheduler PID to the output.
 
-## __cylc-8.1.3 (<span actions:bind='release-date'>Upcoming</span>)__
+[#5461](https://github.com/cylc/cylc-flow/pull/5461) - preserve colour
+formatting when starting workflows in distributed mode using `run hosts`.
+
+[#5291](https://github.com/cylc/cylc-flow/pull/5291) - re-implement old-style
+clock triggers as wall_clock xtriggers.
+
+[#5439](https://github.com/cylc/cylc-flow/pull/5439) - Small CLI short option chages:
+Add the `-n` short option for `--workflow-name` to `cylc vip`; rename the `-n`
+short option for `--no-detach` to `-N`; add `-r` as a short option for
+`--run-name`.
+
+### Fixes
+[#5328](https://github.com/cylc/cylc-flow/pull/5328) -
+Efficiency improvements to reduce task management overheads on the Scheduler.
+
+
+-------------------------------------------------------------------------------
+## __cylc-8.1.3 (<span actions:bind='release-date'>Released 2023-04-27</span>)__
 
 ### Enhancements
 
+[#5475](https://github.com/cylc/cylc-flow/pull/5475) - much faster computation
+of the visualization window around active tasks (at the cost, for now, of not
+showing non-active "cousin" nodes).
+
 [#5453](https://github.com/cylc/cylc-flow/pull/5453) - `cylc cat-log` can now
 list and view workflow log files including install logs and workflow
 configuration files.
 
 ### Fixes
 
+[#5495](https://github.com/cylc/cylc-flow/pull/5495) - Fix bug that could cause
+invalid parent tasks to appear in the UI datastore.
+
+[#5334](https://github.com/cylc/cylc-flow/pull/5334) - Apply graph prerequisite
+changes to already-spawned tasks after reload or restart.
+
+[5466](https://github.com/cylc/cylc-flow/pull/5466) - Don't generate duplicate
+prerequisites from recurrences with coincident points.
+
 [5450](https://github.com/cylc/cylc-flow/pull/5450) - Validation provides
 better error messages if [sections] and settings are mixed up in a
 configuration.

CONTRIBUTING.md

@@ -1,28 +1,37 @@
 # Cylc: How to Contribute
 
-## Report Bugs
+Thanks for you interest in the Cylc project!
 
-Report bugs by opening an issue at [Cylc Issues on
-Github](https://github.com/cylc/cylc-flow/issues). Give the Cylc version
-affected by the bug (you should test the latest release if possible) and a
-recipe to reproduce the problem.
+Contributions are welcome, please open an issue to discuss changes before
+raising a pull request.
 
-## Request Enhancements
+You can also get in touch via:
+
+* The developers chat: [![chat](https://img.shields.io/matrix/cylc-general:matrix.org)](https://matrix.to/#/#cylc-general:matrix.org)
+* The forum: [![forum](https://img.shields.io/discourse/https/cylc.discourse.group/posts.svg)](https://cylc.discourse.group/)
+
+
+## New Contributors
+
+Please read the [CLA](#contributor-licence-agreement-and-certificate-of-origin).
+
+Please add your name to the
+[Code Contributors](#code-contributors) section of this file as part of your
+first Pull Request (for each Cylc repository you contribute to).
 
-Request enhancements by opening an issue at [Cylc Issues @
-Github](https://github.com/cylc/cylc-flow/issues). Describe your use case in
-detail.
 
 ## Contribute Code
 
-All contributions to Cylc are made via Pull Requests against the *master*
-branch of [cylc/cylc-flow](https://github.com/cylc/cylc-flow). Non-trivial
-developments must be discussed and agreed in advance in a [Cylc
-Issue](https://github.com/cylc/cylc-flow/issues) as the team may not be able to
-consider large changes that appear out of the blue. New contributors should
-add their details to the [Code Contributors](#code-contributors) section of
-this file as part of their first Pull Request, and reviewers are responsible
-for checking this before merging the new branch.
+**Enhancements** are made on the `master` branch.
+
+**Bugfixes** are made on the branch of the same name as the issue's milestone.
+E.G. if the issue is on the `8.0.x` milestone, branch off of `8.0.x` to
+develop your bugfix, then raise the pull request against the `8.0.x` branch.
+We will later merge the `8.0.x` branch into `master`.
+
+Feel free to ask questions on the issue or developers chat if unsure about
+anything.
+
 
 ## Code Contributors
 
@@ -73,11 +82,13 @@ requests_).
  - Ronnie Dutta
  - John Haiducek
  - (Andrew Huang)
+ - Cheng Da
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version
 control logs or otherwise.)
 
+
 ## Contributor Licence Agreement and Certificate of Origin
 
 By making a contribution to this project, I certify that:

README.md

@@ -53,6 +53,12 @@ cylc play example
 cylc tui example
 ```
 
+### The Cylc Ecosystem
+
+- [cylc-flow](https://github.com/cylc/cylc-flow) - The core Cylc Scheduler for defining and running workflows.
+- [cylc-uiserver](https://github.com/cylc/cylc-uiserver) - The web-based Cylc graphical user interface for monitoring and controlling workflows.
+- [cylc-rose](https://github.com/cylc/cylc-rose) - Provides integration with [Rose](http://metomi.github.io/rose/).
+
 ### Migrating From Cylc 7
 
 [Migration Guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html)

conda-environment.yml

@@ -2,7 +2,6 @@ name: cylc-dev
 channels:
   - conda-forge
 dependencies:
-  - aiofiles >=0.7.0,<0.8.0
   - ansimarkup >=1.0.0
   - async-timeout>=3.0.0
   - colorama >=0.4,<1.0
@@ -15,7 +14,7 @@ dependencies:
   - protobuf >=4.21.2,<4.22.0
   - psutil >=5.6.0
   - python
-  - pyzmq >=22,<23
+  - pyzmq >=22
   - setuptools >=49, <67
   - importlib_metadata # [py<3.8]
   - urwid >=2,<3

cylc/flow/__init__.py

@@ -46,7 +46,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.1.3.dev'
+__version__ = '8.2.0.dev'
 
 
 def iter_entry_points(entry_point_name):

cylc/flow/async_util.py

@@ -16,12 +16,11 @@
 """Utilities for use with asynchronous code."""
 
 import asyncio
+from functools import partial, wraps
 import os
 from pathlib import Path
 from typing import List, Union
 
-from aiofiles.os import wrap  # type: ignore[attr-defined]
-
 from cylc.flow import LOG
 
 
@@ -60,12 +59,12 @@ class _AsyncPipe:
     """
 
     def __init__(
-            self,
-            func,
-            args=None,
-            kwargs=None,
-            filter_stop=True,
-            preserve_order=True
+        self,
+        func,
+        args=None,
+        kwargs=None,
+        filter_stop=True,
+        preserve_order=True
     ):
         self.func = func
         self.args = args or ()
@@ -393,9 +392,6 @@ def _pipe(func):
         return _pipe
 
 
-async_listdir = wrap(os.listdir)
-
-
 async def scandir(path: Union[Path, str]) -> List[Path]:
     """Asynchronous directory listing (performs os.listdir in an executor)."""
     return [
@@ -449,3 +445,23 @@ async def unordered_map(coroutine, iterator):
         )
         for task in done:
             yield task._args, task.result()
+
+
+def make_async(fcn):
+    """Make a synchronous function async by running it in an executor.
+
+    The default asyncio executor is the ThreadPoolExecutor so this essentially
+    syntactic sugar for running the wrapped function in a thread.
+    """
+    @wraps(fcn)
+    async def _fcn(*args, executor=None, **kwargs):
+        nonlocal fcn
+        return await asyncio.get_event_loop().run_in_executor(
+            executor,
+            partial(fcn, *args, **kwargs),
+        )
+
+    return _fcn
+
+
+async_listdir = make_async(os.listdir)