Merge branch 'master' into 4051

.github/workflows/2_auto_publish_release.yml

@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/[email protected].7
+      uses: pypa/[email protected].8
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

CHANGES.md

@@ -21,13 +21,22 @@ issue which could cause jobs to fail if this variable became too long.
 
 ### Enhancements
 
+[#5992](https://github.com/cylc/cylc-flow/pull/5992) -
+Before trying to reload the workflow definition, the scheduler will
+now wait for preparing tasks to submit, and pause the workflow. 
+After successful reload the scheduler will unpause the workflow.
+
 -[#5605](https://github.com/cylc/cylc-flow/pull/5605) - A shorthand for defining
 -a list of strings - Before: `cylc command -s "X=['a', 'bc', 'd']"` - After:
 -`cylc command -z X=a,bc,d`.
 
 [#5537](https://github.com/cylc/cylc-flow/pull/5537) - Allow parameters
 in family names to be split, e.g. `<foo>FAM<bar>`.
 
+[#5589](https://github.com/cylc/cylc-flow/pull/5589) - Move to workflow
+directory during file parsing, to give the template processor access to
+workflow files.
+
 [#5405](https://github.com/cylc/cylc-flow/pull/5405) - Improve scan command
 help, and add scheduler PID to the output.
 
@@ -71,10 +80,18 @@ Various enhancements to `cylc lint`:
 [#5616](https://github.com/cylc/cylc-flow/pull/5616) -
 Improve PBS support for job IDs with trailing components.
 
+[#5619](https://github.com/cylc/cylc-flow/pull/5619) -
+Fix an issue where the `task_pool` table in the database wasn't being updated
+in a timely fashion when tasks completed.
+
 [#5606](https://github.com/cylc/cylc-flow/pull/5606) -
 Task outputs and messages are now validated to avoid conflicts with built-in
 outputs, messages, qualifiers and Cylc keywords.
 
+[#5614](https://github.com/cylc/cylc-flow/pull/5614) -
+Fix a bug in Cylc 7 compatibility mode where tasks running in the `none` flow
+(e.g. via `cylc trigger --flow=none`) would trigger downstream tasks.
+
 [#5604](https://github.com/cylc/cylc-flow/pull/5604) -
 Fix a possible issue where workflows started using
 `cylc play --start-cycle-point` could hang during startup.
@@ -93,6 +110,13 @@ mode before running pre-configure plugins.
 Permit commas in xtrigger arguments and fix minor issues with the parsing of
 xtrigger function signatures.
 
+[#5618](https://github.com/cylc/cylc-flow/pull/5618) -
+Fix a bug when rapidly issuing the same/opposite commands e.g. pausing &
+resuming a workflow.
+
+[#5625](https://github.com/cylc/cylc-flow/pull/5625) - Exclude `setuptools`
+version (v67) which results in dependency check failure with editable installs.
+
 ## __cylc-8.1.4 (<span actions:bind='release-date'>Released 2023-05-04</span>)__
 
 ### Fixes

conda-environment.yml

@@ -15,7 +15,7 @@ dependencies:
   - psutil >=5.6.0
   - python
   - pyzmq >=22
-  - setuptools >=49
+  - setuptools >=49,!=67.*
   - importlib_metadata # [py<3.8]
   - urwid >=2,<3
   - tomli >=2 # [py<3.11]

cylc/flow/cfgspec/globalcfg.py

@@ -1024,11 +1024,11 @@ def default_for(
                     Alternative location for the run dir.
 
                     If specified, the workflow run directory will
-                    be created in ``<this-path>/cylc-run/<workflow-name>``
+                    be created in ``<this-path>/cylc-run/<workflow-id>``
                     and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-name>``.
+                    ``$HOME/cylc-run/<workflow-id>``.
                     If not specified the workflow run directory will be created
-                    in ``$HOME/cylc-run/<workflow-name>``.
+                    in ``$HOME/cylc-run/<workflow-id>``.
                     All the workflow files and the ``.service`` directory get
                     installed into this directory.
 
@@ -1038,23 +1038,23 @@ def default_for(
                     Alternative location for the log dir.
 
                     If specified the workflow log directory will be created in
-                    ``<this-path>/cylc-run/<workflow-name>/log`` and a
+                    ``<this-path>/cylc-run/<workflow-id>/log`` and a
                     symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-name>/log``. If not specified
+                    ``$HOME/cylc-run/<workflow-id>/log``. If not specified
                     the workflow log directory will be created in
-                    ``$HOME/cylc-run/<workflow-name>/log``.
+                    ``$HOME/cylc-run/<workflow-id>/log``.
 
                     .. versionadded:: 8.0.0
                 """)
                 Conf('share', VDR.V_STRING, None, desc="""
                     Alternative location for the share dir.
 
                     If specified the workflow share directory will be
-                    created in ``<this-path>/cylc-run/<workflow-name>/share``
+                    created in ``<this-path>/cylc-run/<workflow-id>/share``
                     and a symbolic link will be created from
-                    ``<$HOME/cylc-run/<workflow-name>/share``. If not specified
+                    ``<$HOME/cylc-run/<workflow-id>/share``. If not specified
                     the workflow share directory will be created in
-                    ``$HOME/cylc-run/<workflow-name>/share``.
+                    ``$HOME/cylc-run/<workflow-id>/share``.
 
                     .. versionadded:: 8.0.0
                 """)
@@ -1063,23 +1063,23 @@ def default_for(
 
                     If specified the workflow share/cycle directory
                     will be created in
-                    ``<this-path>/cylc-run/<workflow-name>/share/cycle``
+                    ``<this-path>/cylc-run/<workflow-id>/share/cycle``
                     and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-name>/share/cycle``. If not
+                    ``$HOME/cylc-run/<workflow-id>/share/cycle``. If not
                     specified the workflow share/cycle directory will be
-                    created in ``$HOME/cylc-run/<workflow-name>/share/cycle``.
+                    created in ``$HOME/cylc-run/<workflow-id>/share/cycle``.
 
                     .. versionadded:: 8.0.0
                 """)
                 Conf('work', VDR.V_STRING, None, desc="""
                     Alternative directory for the work dir.
 
                     If specified the workflow work directory will be created in
-                    ``<this-path>/cylc-run/<workflow-name>/work`` and a
+                    ``<this-path>/cylc-run/<workflow-id>/work`` and a
                     symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-name>/work``. If not specified
+                    ``$HOME/cylc-run/<workflow-id>/work``. If not specified
                     the workflow work directory will be created in
-                    ``$HOME/cylc-run/<workflow-name>/work``.
+                    ``$HOME/cylc-run/<workflow-id>/work``.
 
                     .. versionadded:: 8.0.0
                 """)

cylc/flow/cfgspec/workflow.py

@@ -1530,7 +1530,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
                 The items in this section reflect
                 options and defaults of the ``cylc workflow-state`` command,
-                except that the target workflow name and the
+                except that the target workflow ID and the
                 ``--task``, ``--cycle``, and ``--status`` options are
                 taken from the graph notation.
 

cylc/flow/clean.py

@@ -103,15 +103,15 @@ async def get_contained_workflows(partial_id) -> List[str]:
     )
 
 
-def _clean_check(opts: 'Values', reg: str, run_dir: Path) -> None:
+def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None:
     """Check whether a workflow can be cleaned.
 
     Args:
-        reg: Workflow name.
+        id_: Workflow name.
         run_dir: Path to the workflow run dir on the filesystem.
     """
-    validate_workflow_name(reg)
-    reg = os.path.normpath(reg)
+    validate_workflow_name(id_)
+    id_ = os.path.normpath(id_)
     # Thing to clean must be a dir or broken symlink:
     if not run_dir.is_dir() and not run_dir.is_symlink():
         raise FileNotFoundError(f"No directory to clean at {run_dir}")
@@ -124,10 +124,10 @@ def _clean_check(opts: 'Values', reg: str, run_dir: Path) -> None:
         # about contact file.
         return
     try:
-        detect_old_contact_file(reg)
+        detect_old_contact_file(id_)
     except ContactFileExists as exc:
         raise ServiceFileError(
-            f"Cannot clean running workflow {reg}.\n\n{exc}"
+            f"Cannot clean running workflow {id_}.\n\n{exc}"
         )
 
 
@@ -336,7 +336,7 @@ def _clean_using_glob(
 
 
 def remote_clean(
-    reg: str,
+    id_: str,
     platform_names: Iterable[str],
     rm_dirs: Optional[List[str]] = None,
     timeout: str = '120'
@@ -345,7 +345,7 @@ def remote_clean(
     (skip localhost), given a set of platform names to look up.
 
     Args:
-        reg: Workflow name.
+        id_: Workflow name.
         platform_names: List of platform names to look up in the global
             config, in order to determine the install targets to clean on.
         rm_dirs: Sub dirs to remove instead of the whole run dir.
@@ -356,18 +356,18 @@ def remote_clean(
             get_install_target_to_platforms_map(platform_names))
     except PlatformLookupError as exc:
         raise PlatformLookupError(
-            f"Cannot clean {reg} on remote platforms as the workflow database "
+            f"Cannot clean {id_} on remote platforms as the workflow database "
             f"is out of date/inconsistent with the global config - {exc}")
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(
-        _remote_clean_cmd, reg=reg, rm_dirs=rm_dirs, timeout=timeout
+        _remote_clean_cmd, id_=id_, rm_dirs=rm_dirs, timeout=timeout
     )
     for target, platforms in install_targets_map.items():
         if target == get_localhost_install_target():
             continue
         shuffle(platforms)
         LOG.info(
-            f"Cleaning {reg} on install target: "
+            f"Cleaning {id_} on install target: "
             f"{platforms[0]['install target']}"
         )
         # Issue ssh command:
@@ -417,13 +417,13 @@ def remote_clean(
     if failed_targets:
         for target, excp in failed_targets.items():
             LOG.error(
-                f"Could not clean {reg} on install target: {target}\n{excp}"
+                f"Could not clean {id_} on install target: {target}\n{excp}"
             )
-        raise CylcError(f"Remote clean failed for {reg}")
+        raise CylcError(f"Remote clean failed for {id_}")
 
 
 def _remote_clean_cmd(
-    reg: str,
+    id_: str,
     platform: Dict[str, Any],
     rm_dirs: Optional[List[str]],
     timeout: str
@@ -433,7 +433,7 @@ def _remote_clean_cmd(
     Call "cylc clean --local-only" over ssh and return the subprocess.
 
     Args:
-        reg: Workflow name.
+        id_: Workflow name.
         platform: Config for the platform on which to remove the workflow.
         rm_dirs: Sub dirs to remove instead of the whole run dir.
         timeout: Number of seconds to wait before cancelling the command.
@@ -443,10 +443,10 @@ def _remote_clean_cmd(
 
     """
     LOG.debug(
-        f"Cleaning {reg} on install target: {platform['install target']} "
+        f"Cleaning {id_} on install target: {platform['install target']} "
         f"(using platform: {platform['name']})"
     )
-    cmd = ['clean', '--local-only', reg]
+    cmd = ['clean', '--local-only', id_]
     if rm_dirs is not None:
         for item in rm_dirs:
             cmd.extend(['--rm', item])

cylc/flow/config.py

@@ -59,11 +59,11 @@
 from cylc.flow.cycling.iso8601 import ingest_time, ISO8601Interval
 from cylc.flow.exceptions import (
     CylcError,
-    WorkflowConfigError,
+    InputError,
     IntervalParsingError,
-    TaskDefError,
     ParamExpandError,
-    InputError
+    TaskDefError,
+    WorkflowConfigError,
 )
 import cylc.flow.flags
 from cylc.flow.graph_parser import GraphParser
@@ -209,6 +209,8 @@ def dequote(string):
         'foo'
         >>> dequote('"f')
         '"f'
+        >>> dequote('f')
+        'f'
 
     """
     if len(string) < 2:

cylc/flow/install.py

@@ -547,7 +547,7 @@ def detect_flow_exists(
     """Returns True if installed flow already exists.
 
     Args:
-        run_path_base: Absolute path of workflow directory,
+        run_path_base: Absolute path of the parent of the workflow's run dir,
             i.e ~/cylc-run/<workflow_name>
         numbered: If True, will detect if numbered runs exist. If False, will
             detect if non-numbered runs exist, i.e. runs installed
@@ -583,8 +583,8 @@ def check_nested_dirs(
             install dirs.
 
     Raises:
-        WorkflowFilesError if reg dir is nested inside a run dir, or an
-            install dirs are nested.
+        WorkflowFilesError if run_dir is nested inside an existing run dir,
+            or install dirs are nested.
     """
     if install_dir is not None:
         install_dir = Path(os.path.normpath(install_dir))

cylc/flow/network/__init__.py

@@ -67,7 +67,7 @@ def get_location(workflow: str) -> Tuple[str, int, int]:
     NB: if it fails to load the workflow contact file, it will exit.
 
     Args:
-        workflow: workflow name
+        workflow: workflow ID
     Returns:
         Tuple (host name, port number, publish port number)
     Raises:

cylc/flow/network/authentication.py

@@ -24,12 +24,12 @@
     remove_keys_on_server)
 
 
-def key_housekeeping(reg, platform=None, create=True):
+def key_housekeeping(id_, platform=None, create=True):
 
     """Clean any existing authentication keys and create new ones.
         If create is set to false, keys will only be cleaned from
         server."""
-    workflow_srv_dir = get_workflow_srv_dir(reg)
+    workflow_srv_dir = get_workflow_srv_dir(id_)
     keys = {
         "client_public_key": KeyInfo(
             KeyType.PUBLIC,

cylc/flow/network/client_factory.py

@@ -46,7 +46,7 @@ def get_runtime_client(
 
         Args:
             comm_method: communication method
-            workflow: workflow name
+            workflow: workflow ID
     """
     if comms_method == CommsMeth.SSH:
         from cylc.flow.network.ssh_client import WorkflowRuntimeClient

cylc/flow/network/scan.py

@@ -536,21 +536,13 @@ async def workflow_params(flow):
     Requires:
         * is_active(True)
     """
-    params = {}
-
-    def _callback(_, entry):
-        nonlocal params
-        key, value = entry
-        params[key] = value
-
     # NOTE: use the public DB for reading
     # (only the scheduler process/thread should access the private database)
     db_file = Path(get_workflow_run_dir(
         flow['name'], WorkflowFiles.LogDir.DIRNAME, WorkflowFiles.LogDir.DB
     ))
     if db_file.exists():
         with CylcWorkflowDAO(db_file, is_public=True) as dao:
-            dao.select_workflow_params(_callback)
-            flow['workflow_params'] = params
+            flow['workflow_params'] = dict(dao.select_workflow_params())
 
     return flow

cylc/flow/parsec/fileparse.py

@@ -401,6 +401,12 @@ def read_and_proc(
     fpath = _get_fpath_for_source(fpath, opts)
     fdir = os.path.dirname(fpath)
 
+    odir = os.getcwd()
+
+    # Move to the file location to give the template processor easy access to
+    # other files in the workflow directory (whether source or installed).
+    os.chdir(fdir)
+
     # Allow Python modules in lib/python/ (e.g. for use by Jinja2 filters).
     workflow_lib_python = os.path.join(fdir, "lib", "python")
     if (
@@ -494,6 +500,8 @@ def read_and_proc(
     if do_contin:
         flines = _concatenate(flines)
 
+    os.chdir(odir)
+
     # return rstripped lines
     return [fl.rstrip() for fl in flines]