Merge remote-tracking branch 'upstream/master' into HEAD

* upstream/master: (350 commits) Apply suggestions from code review [skip ci] Added missing union type import (cylc#5906) set CYLC_DEBUG env var when set-verbosity DEBUG (cylc#5854) coverage: exclude report-timings and set 90% as the lower limit lint: Warn users that `CYLC_VERSION={{CYLC_VERSION}}` is deprecated (cylc#5890) Add a new entry_point for xtriggers (cylc#5831) Update copyright year Tests: fix cached datetime arithmetic contamination between tests using different calendar modes Replace cyclers functional reftests with integration reftests Simplify integration reftest Replace pre-initial functional reftests with integration reftests Integration tests: add a simpler reftest fixture auto update syntax files id: catch traceback in ID parsing actions: downgrade macos runners to macos 11 (cylc#5892) Feat.lint obsolete vars (cylc#5879) dump: restrict window to n=0 (cylc#5600) Add chained offset logic for FCP (cylc#5885) Test.example replace a reftest (cylc#5860) GH Actions artifact name fixes ...
wxtim · Jan 9, 2024 · ca49ba8 · ca49ba8
2 parents 9643892 + df350cd
commit ca49ba8
Show file tree

Hide file tree

Showing 407 changed files with 17,325 additions and 4,847 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -17,18 +17,19 @@ coverage:
     project:
       default:
         # commits below this threshold will be marked as failed
-        target: '85%'
+        target: '92%'
         # how much we allow the coverage to drop
         threshold: '2%'
     patch:
       default:
-        target: '95%'
+        target: '97%'
         threshold: '5%'
 
 # files to ignore
 ignore:
   - "tests/**"
   - "ws_messages_pb2.py"
+  - "cylc/flow/scripts/report_timings.py"
 
 flag_management:
   default_rules:

diff --git a/.coveragerc b/.coveragerc
@@ -16,6 +16,7 @@ omit =
     tests/*
     */cylc/flow/*_pb2.py
     cylc/flow/etc/*
+    cylc/flow/scripts/report_timings.py
 parallel = True
 source = ./cylc
 timid = False

diff --git a/.dockerignore b/.dockerignore
@@ -9,3 +9,4 @@ __pycache__
 !cylc
 !dockerfiles/*
 !setup*
+!MANIFEST.in
diff --git a/.github/workflows/1_create_release_pr.yml b/.github/workflows/1_create_release_pr.yml
@@ -21,7 +21,7 @@ jobs:
         uses: cylc/release-actions/stage-1/sanitize-inputs@v1
 
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE_REF }}
           fetch-depth: 0  # need to fetch all commits to check contributors
@@ -30,10 +30,9 @@ jobs:
         uses: cylc/release-actions/check-shortlog@v1
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          # return to 3.x once compatible with 3.10+ (pyuv)
-          python-version: '3.9'
+          python-version: '3.x'
 
       - name: Create & checkout PR branch
         uses: cylc/release-actions/stage-1/checkout-pr-branch@v1
@@ -44,15 +43,14 @@ jobs:
           init-file: 'cylc/flow/__init__.py'
           pypi-package-name: 'cylc-flow'
 
-      - name: Update "released on" date in changelog
-        continue-on-error: true
-        uses: cylc/release-actions/stage-1/update-changelog-release-date@v1
-        with:
-          changelog-file: 'CHANGES.md'
-
       - name: Test build
         uses: cylc/release-actions/build-python-package@v1
 
+      - name: Generate changelog
+        run: |
+          python3 -m pip install -q towncrier
+          towncrier build --yes
+
       - name: Create pull request
         uses: cylc/release-actions/stage-1/create-release-pr@v1
         with:

diff --git a/.github/workflows/2_auto_publish_release.yml b/.github/workflows/2_auto_publish_release.yml
@@ -22,15 +22,14 @@ jobs:
     steps:
 
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         ref: ${{ env.MERGE_SHA }}
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        # return to 3.x once compatible with 3.10+ (pyuv)
-        python-version: '3.9'
+        python-version: '3.x'
 
     - name: Get the version number
       uses: cylc/release-actions/stage-2/get-version-from-pr@v1
@@ -39,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/[email protected].5
+      uses: pypa/[email protected].11
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

diff --git a/.github/workflows/bash.yml b/.github/workflows/bash.yml
@@ -44,7 +44,7 @@ jobs:
           - '5.0'
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Docker container
         run: |
@@ -85,7 +85,7 @@ jobs:
 
       - name: Upload artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: Upload cylc-run artifact
+          name: 'cylc-run (bash-${{ matrix.bash-version }})'
           path: cylc-run
diff --git a/.github/workflows/branch_sync.yml b/.github/workflows/branch_sync.yml
@@ -0,0 +1,20 @@
+name: Sync PR
+
+on:
+  push:
+    branches:
+      - '8.*.x'
+  schedule:
+    - cron: '33 04 * * 1-5' # 04:33 UTC Mon-Fri
+  workflow_dispatch:
+    inputs:
+      head_branch:
+        description: Branch to merge into master
+        required: true
+
+jobs:
+  sync:
+    uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1
+    with:
+      head_branch: ${{ inputs.head_branch }}
+    secrets: inherit
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,43 @@
+name: Build
+
+# build the project whenever the configuration is changed
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'README.md'       # check markdown is valid
+      - 'MANIFEST.in'     # check packaging
+      - 'pyproject.toml'  # check build config
+      - 'setup.cfg'       # check deps and project config
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 10
+    strategy:
+      matrix:
+        os: ['ubuntu-latest']
+        python: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        include:
+          - os: 'macos-latest'
+            python: '3.7'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Build
+        uses: cylc/release-actions/build-python-package@v1
+
+      - name: Inspect
+        run: |
+          unzip -l dist/*.whl | tee files
+          grep 'cylc_flow.*.dist-info/COPYING' files
+          grep 'cylc/flow/py.typed' files
+          grep 'cylc/flow/etc' files
+          grep 'cylc/flow/etc/cylc-completion.bash' files
diff --git a/.github/workflows/shortlog.yml b/.github/workflows/shortlog.yml
@@ -14,7 +14,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0  # need to fetch all commits to check contributors
 

diff --git a/.github/workflows/test_conda-build.yml b/.github/workflows/test_conda-build.yml
@@ -19,7 +19,7 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: checkout cylc-flow
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: build conda env
         run: |

diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
@@ -17,43 +17,39 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
-      fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs
+      fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.10', '3.11', '3']
         include:
-          - os: 'macos-latest'
-            python-version: '3.7'
+          # mac os test
+          - os: 'macos-11'
+            python-version: '3.7'  # oldest supported version
+
+          # non-utc timezone test
+          - os: 'ubuntu-latest'
+            python-version: '3.9'  # not the oldest, not the most recent version
+            time-zone: 'XXX-09:35'
+
     env:
+      # Use non-UTC time zone
+      TZ: ${{ matrix.time-zone }}
       PYTEST_ADDOPTS: --cov --cov-append -n 5 --color=yes
+
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Brew Install
-        if: startsWith(matrix.os, 'macos')
-        run: |
-          # speed up install (https://docs.brew.sh/Manpage#environment)
-          export HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1
-          echo "[command]brew update"
-          brew update
-          echo "[command]brew install ..."
-          brew install bash coreutils
-          # add GNU coreutils and sed to the user PATH
-          # (see instructions in brew install output)
-          echo "$(brew --prefix)/opt/coreutils/libexec/gnubin" \
-            >> "${GITHUB_PATH}"
-
       - name: Apt-Get Install
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           sudo apt-get update
-          sudo apt-get install -y shellcheck sqlite3
+          sudo apt-get install -y sqlite3
 
       - name: Install
         run: |
@@ -62,40 +58,19 @@ jobs:
       - name: Configure git  # Needed by the odd test
         uses: cylc/release-actions/configure-git@v1
 
-      - name: Style
-        if: startsWith(matrix.os, 'ubuntu')
-        run: |
-          flake8
-          etc/bin/shellchecker
-
-      - name: Typing
-        if: startsWith(matrix.os, 'ubuntu')
-        run: mypy
-
-      - name: Doctests
-        timeout-minutes: 4
-        run: |
-          pytest cylc/flow
-
       - name: Unit Tests
-        timeout-minutes: 4
+        timeout-minutes: 5
         run: |
-          pytest tests/unit
-
-      - name: Bandit
-        if: ${{ matrix.python-version == '3.7' }}
-        # https://github.com/PyCQA/bandit/issues/658
-        run: |
-          bandit -r --ini .bandit cylc/flow
+          pytest cylc/flow tests/unit
 
       - name: Integration Tests
         timeout-minutes: 6
         run: |
           pytest tests/integration
 
-      - name: Upload artifact
+      - name: Upload failed tests artifact
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: cylc-run (${{ matrix.os }} py-${{ matrix.python-version }})
           path: ~/cylc-run/
@@ -105,15 +80,73 @@ jobs:
           coverage xml
           coverage report
 
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage_${{ matrix.os }}_py-${{ matrix.python-version }}
+          path: coverage.xml
+          retention-days: 7
+
+  lint:
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 10
+    steps:
+      - name: Apt-Get Install
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # note: exclude python 3.10+ from mypy checks as these produce false
+      # positives in installed libraries for python 3.7
+      - name: Configure Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+
+      - name: Install
+        run: |
+          pip install -e ."[tests]"
+
+      - name: Flake8
+        run: flake8
+
+      - name: Bandit
+        run: |
+          bandit -r --ini .bandit cylc/flow
+
+      - name: Shellchecker
+        run: etc/bin/shellchecker
+
+      - name: MyPy
+        run: mypy
+
+      - name: Towncrier
+        run: towncrier build --draft
+
+      - name: Linkcheck
+        run: pytest -m linkcheck --dist=load --color=yes -n 10 tests/unit/test_links.py
+
+  codecov:
+    needs: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Download coverage artifacts
+        uses: actions/download-artifact@v4
+
       - name: Codecov upload
         uses: codecov/codecov-action@v3
         with:
-          name: '${{ github.workflow }} ${{ matrix.os }} py-${{ matrix.python-version }}'
+          name: ${{ github.workflow }}
           flags: fast-tests
           fail_ci_if_error: true
           verbose: true
-          token: ${{ secrets.CODECOV_TOKEN }} # Token not required for public repos, but might reduce chance of random 404 error?
-
-      - name: Linkcheck
-        if: startsWith(matrix.python-version, '3.10')
-        run: pytest -m linkcheck --dist=load tests/unit
+          # Token not required for public repos, but avoids upload failure due
+          # to rate-limiting (but not for PRs opened from forks)
+          token: ${{ secrets.CODECOV_TOKEN }}
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,3 +9,4 @@ __pycache__ @@
     !cylc
     !dockerfiles/*
     !setup*
+    !MANIFEST.in