diff --git a/changes.d/6345.fix.md b/changes.d/6345.fix.md new file mode 100644 index 00000000000..a629eb6c8ed --- /dev/null +++ b/changes.d/6345.fix.md @@ -0,0 +1 @@ +Fix duplicate job submissions of tasks in the preparing state before reload. diff --git a/cylc/flow/task_job_mgr.py b/cylc/flow/task_job_mgr.py index e8b77f14d85..d8de2ef7bb2 100644 --- a/cylc/flow/task_job_mgr.py +++ b/cylc/flow/task_job_mgr.py @@ -537,6 +537,12 @@ def submit_task_jobs(self, workflow, itasks, curve_auth, stdin_files = [] job_log_dirs = [] for itask in itasks_batch: + if not itask.waiting_on_job_prep: + # Avoid duplicate job submissions when flushing + # preparing tasks before a reload. See + # https://github.com/cylc/cylc-flow/pull/6345 + continue + if remote_mode: stdin_files.append( os.path.expandvars( @@ -555,8 +561,11 @@ def submit_task_jobs(self, workflow, itasks, curve_auth, # write flag so that subsequent manual retrigger will # generate a new job file. itask.local_job_file_path = None - itask.waiting_on_job_prep = False + + if not job_log_dirs: + continue + self.proc_pool.put_command( SubProcContext( self.JOBS_SUBMIT, diff --git a/tests/functional/reload/28-preparing.t b/tests/functional/reload/28-preparing.t new file mode 100644 index 00000000000..08f4ba911ac --- /dev/null +++ b/tests/functional/reload/28-preparing.t @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# Test for duplicate job submissions when preparing tasks get flushed +# prior to reload - see https://github.com/cylc/cylc-flow/pull/6345 + +. "$(dirname "$0")/test_header" +set_test_number 4 + +# Strap the process pool size down to 1, so that the first task is stuck +# in the preparing state until the startup event handler finishes. + +create_test_global_config "" " +[scheduler] + process pool size = 1 +" + +# install and play the workflow, then reload it and wait for it to finish. +install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" +run_ok "${TEST_NAME_BASE}-vip" cylc validate "${WORKFLOW_NAME}" +run_ok "${TEST_NAME_BASE}-reload" cylc play "${WORKFLOW_NAME}" +run_ok "${TEST_NAME_BASE}-reload" cylc reload "${WORKFLOW_NAME}" +poll_grep_workflow_log -F 'INFO - DONE' + +# check that task `foo` was only submitted once. +count_ok "1/foo.*submitted to" "${WORKFLOW_RUN_DIR}/log/scheduler/log" 1 + +purge diff --git a/tests/functional/reload/28-preparing/flow.cylc b/tests/functional/reload/28-preparing/flow.cylc new file mode 100644 index 00000000000..27a9f1013e9 --- /dev/null +++ b/tests/functional/reload/28-preparing/flow.cylc @@ -0,0 +1,8 @@ +[scheduler] + [[events]] + startup handlers = "sleep 10; echo" +[scheduling] + [[graph]] + R1 = "foo => bar" +[runtime] + [[foo, bar]]