Merge remote-tracking branch 'upstream/8.1.x' into dont_get_platform_…

…too_soon * upstream/8.1.x: (46 commits) Bump dev version (cylc#5518) Prepare release 8.1.4 cat-log: fix issues with file listing Update changelog Fix `eval_host()` localhost bug & plug testing gap Bump dev version (cylc#5501) Prepare release 8.1.3 Ignore off-sequence parents, for datastore. (cylc#5495) Avoid duplicate prerequisites from multiple recurrences. (cylc#5466) tests/f: fix events/11 swarm: convert cylc-dev Docker image to run on ubuntu:latest fix changelog Apply prerequisite changes to spawned tasks, on reload & restart (cylc#5334) cat-log: don't list directories which don't exist (cylc#5488) task_job_mgr: move compute out of open call manylinux tests: update to ubuntu-20.04 Update changelog Address review Fix.platform is regex remote tidy fail (cylc#5445) parsec: better error messages for section/setting mixups ...
wxtim · May 4, 2023 · 42205eb · 42205eb
2 parents f0dc1b1 + 882b84c
commit 42205eb
Show file tree

Hide file tree

Showing 72 changed files with 3,008 additions and 643 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -9,3 +9,4 @@ __pycache__
 !cylc
 !dockerfiles/*
 !setup*
+!MANIFEST.in
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,43 @@
+name: Build
+
+# build the project whenever the configuration is changed
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'README.md'       # check markdown is valid
+      - 'MANIFEST.in'     # check packaging
+      - 'pyproject.toml'  # check build config
+      - 'setup.cfg'       # check deps and project config
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      matrix:
+        os: ['ubuntu-latest']
+        python: ['3.7', '3.8', '3.9', '3.10']
+        include:
+          - os: 'macos-latest'
+            python: '3.7'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Build
+        uses: cylc/release-actions/build-python-package@v1
+
+      - name: Inspect
+        run: |
+          unzip -l dist/*.whl | tee files
+          grep 'cylc_flow.*.dist-info/COPYING' files
+          grep 'cylc/flow/py.typed' files
+          grep 'cylc/flow/etc' files
+          grep 'cylc/flow/etc/cylc-completion.bash' files
diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
@@ -35,20 +35,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Brew Install
-        if: startsWith(matrix.os, 'macos')
-        run: |
-          # speed up install (https://docs.brew.sh/Manpage#environment)
-          export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
-          echo "[command]brew update"
-          brew update
-          echo "[command]brew install ..."
-          brew install bash coreutils
-          # add GNU coreutils and sed to the user PATH
-          # (see instructions in brew install output)
-          echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" \
-            >> "${GITHUB_PATH}"
-
       - name: Apt-Get Install
         if: startsWith(matrix.os, 'ubuntu')
         run: |
@@ -93,7 +79,7 @@ jobs:
         run: |
           pytest tests/integration
 
-      - name: Upload artifact
+      - name: Upload failed tests artifact
         if: failure()
         uses: actions/upload-artifact@v3
         with:
@@ -105,15 +91,35 @@ jobs:
           coverage xml
           coverage report
 
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }}
+          path: coverage.xml
+          retention-days: 7
+
+      - name: Linkcheck
+        if: startsWith(matrix.python-version, '3.10')
+        run: pytest -m linkcheck --dist=load tests/unit
+
+  codecov:
+    needs: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v3
+
       - name: Codecov upload
         uses: codecov/codecov-action@v3
         with:
-          name: '${{ github.workflow }} ${{ matrix.os }} py-${{ matrix.python-version }}'
+          name: ${{ github.workflow }}
           flags: fast-tests
           fail_ci_if_error: true
           verbose: true
-          token: ${{ secrets.CODECOV_TOKEN }} # Token not required for public repos, but might reduce chance of random 404 error?
-
-      - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+          # Token not required for public repos, but avoids upload failure due
+          # to rate-limiting (but not for PRs opened from forks)
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
@@ -248,14 +248,13 @@ jobs:
               -exec echo '====== {} ======' \; -exec cat '{}' \;
 
       - name: Set artifact upload name
-        if: failure() && steps.test.outcome == 'failure'
         id: uploadname
         run: |
           # artifact name cannot contain '/' characters
           CID="$(sed 's|/|-|g' <<< "${{ matrix.name || matrix.chunk }}")"
           echo "uploadname=$CID" >> $GITHUB_OUTPUT
 
-      - name: Upload artifact
+      - name: Upload failed tests artifact
         if: failure() && steps.test.outcome == 'failure'
         uses: actions/upload-artifact@v3
         with:
@@ -294,11 +293,31 @@ jobs:
           coverage xml
           coverage report
 
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage_${{ steps.uploadname.outputs.uploadname }}
+          path: coverage.xml
+          retention-days: 7
+
+  codecov:
+    needs: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v3
+
       - name: Codecov upload
         uses: codecov/codecov-action@v3
         with:
-          name: '${{ github.workflow }} ${{ matrix.name }} ${{ matrix.chunk }}'
+          name: ${{ github.workflow }}
           flags: functional-tests
           fail_ci_if_error: true
           verbose: true
-          token: ${{ secrets.CODECOV_TOKEN }} # Token not required for public repos, but might reduce chance of random 404 error?
+          # Token not required for public repos, but avoids upload failure due
+          # to rate-limiting (but not for PRs opened from forks)
+          token: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/test_manylinux.yml b/.github/workflows/test_manylinux.yml
@@ -33,7 +33,7 @@ jobs:
       fail-fast: false
       matrix:
         manylinux: ['1']
-        os: ['ubuntu-18.04']  # run on the oldest linux we have access to
+        os: ['ubuntu-20.04']  # run on the oldest linux we have access to
         python-version: ['3.7', '3.8', '3.9', '3.10']
 
     steps:
@@ -73,7 +73,7 @@ jobs:
             >> _manylinux.py
 
       - name: Install
-        timeout-minutes: 25
+        timeout-minutes: 35
         run: |
           PYTHONPATH="$PWD:$PYTHONPATH" pip install ."[all]"
 
@@ -84,6 +84,6 @@ jobs:
           import cylc.flow.scheduler
 
       - name: Test
-        timeout-minutes: 15
+        timeout-minutes: 5
         run: |
           pytest -n 5
diff --git a/CHANGES.md b/CHANGES.md
@@ -10,20 +10,76 @@ creating a new release entry be sure to copy & paste the span tag with the
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
 -------------------------------------------------------------------------------
-## __cylc-8.1.3 (<span actions:bind='release-date'>Upcoming</span>)__
+## __cylc-8.1.4 (<span actions:bind='release-date'>Released 2023-05-04</span>)__
 
 ### Fixes
 
+[#5514](https://github.com/cylc/cylc-flow/pull/5514) -
+Ensure `cylc cat-log` directory listings always include the `job-activity.log`
+file when present and are able to list submit-failed jobs.
+
+[#5506](https://github.com/cylc/cylc-flow/pull/5506) -
+Fix bug introduced in 8.1.3 where specifying a subshell command for
+`flow.cylc[runtime][<namespace>][remote]host` (e.g. `$(rose host-select)`)
+would always result in localhost.
+
+-------------------------------------------------------------------------------
+## __cylc-8.1.3 (<span actions:bind='release-date'>Released 2023-04-27</span>)__
+
+### Enhancements
+
+[#5475](https://github.com/cylc/cylc-flow/pull/5475) - much faster computation
+of the visualization window around active tasks (at the cost, for now, of not
+showing non-active "cousin" nodes).
+
+[#5453](https://github.com/cylc/cylc-flow/pull/5453) - `cylc cat-log` can now
+list and view workflow log files including install logs and workflow
+configuration files.
+
+### Fixes
+
+[#5495](https://github.com/cylc/cylc-flow/pull/5495) - Fix bug that could cause
+invalid parent tasks to appear in the UI datastore.
+
+[#5334](https://github.com/cylc/cylc-flow/pull/5334) - Apply graph prerequisite
+changes to already-spawned tasks after reload or restart.
+
+[5466](https://github.com/cylc/cylc-flow/pull/5466) - Don't generate duplicate
+prerequisites from recurrences with coincident points.
+
+[5450](https://github.com/cylc/cylc-flow/pull/5450) - Validation provides
+better error messages if [sections] and settings are mixed up in a
+configuration.
+
+[5445](https://github.com/cylc/cylc-flow/pull/5445) - Fix remote tidy
+ bug where install target is not explicit in platform definition.
+
 [5398](https://github.com/cylc/cylc-flow/pull/5398) - Fix platform from
 group selection order bug.
 
+[#5395](https://github.com/cylc/cylc-flow/pull/5395) - Fix bug where workflow
+shuts down if all hosts for all platforms in a platform group are unreachable.
+
 [#5384](https://github.com/cylc/cylc-flow/pull/5384) -
 Fixes `cylc set-verbosity`.
 
-[#5386](https://github.com/cylc/cylc-flow/pull/5386) Fix bug where
+[#5479](https://github.com/cylc/cylc-flow/pull/5479) -
+Fixes `cylc help license`
+
+[#5394](https://github.com/cylc/cylc-flow/pull/5394) -
+Fixes a possible scheduler traceback observed with remote task polling.
+
+[#5386](https://github.com/cylc/cylc-flow/pull/5386) - Fix bug where
 absence of `job name length maximum` in PBS platform settings would cause
 Cylc to crash when preparing the job script.
 
+[#5343](https://github.com/cylc/cylc-flow/pull/5343) - Fix a bug causing
+platform names to be checked as if they were hosts.
+
+[#5359](https://github.com/cylc/cylc-flow/pull/5359) - Fix bug where viewing
+a workflow's log in the GUI or using `cylc cat-log` would prevent `cylc clean`
+from working.
+
 -------------------------------------------------------------------------------
 ## __cylc-8.1.2 (<span actions:bind='release-date'>Released 2023-02-20</span>)__
 

diff --git a/conda-environment.yml b/conda-environment.yml
@@ -17,9 +17,9 @@ dependencies:
   - python
   - pyzmq >=22,<23
   - setuptools >=49, <67
+  - importlib_metadata # [py<3.8]
   - urwid >=2,<3
-  # Add # [py<3.11] for tomli once Python 3.11 Released
-  - tomli >=2
+  - tomli >=2 # [py<3.11]
 
 # optional dependencies
   #- empy >=3.3,<3.4

diff --git a/cylc/flow/__init__.py b/cylc/flow/__init__.py
@@ -46,7 +46,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.1.3.dev'
+__version__ = '8.1.5.dev'
 
 
 def iter_entry_points(entry_point_name):

diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
@@ -1426,7 +1426,8 @@ def default_for(
                        "[remote]retrieve job logs retry delays")}
             ''')
             Conf('tail command template',
-                 VDR.V_STRING, 'tail -n +1 -F %(filename)s', desc=f'''
+                 VDR.V_STRING, 'tail -n +1 --follow=name -F %(filename)s',
+                 desc=f'''
                 A command template (with ``%(filename)s`` substitution) to
                 tail-follow job logs this platform, by ``cylc cat-log``.
 

diff --git a/cylc/flow/config.py b/cylc/flow/config.py
@@ -356,13 +356,6 @@ def __init__(
         except KeyError:
             parameter_templates = {}
 
-        # Check that parameter templates are a section
-        if not hasattr(parameter_templates, 'update'):
-            raise WorkflowConfigError(
-                '[task parameters][templates] is a section. Don\'t use it '
-                'as a parameter.'
-            )
-
         # parameter values and templates are normally needed together.
         self.parameters = (parameter_values, parameter_templates)
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,3 +9,4 @@ __pycache__ @@
     !cylc
     !dockerfiles/*
     !setup*
+    !MANIFEST.in