diff --git a/CHANGES.md b/CHANGES.md index 2acaee55f43..a07d43ed979 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -77,6 +77,9 @@ Various enhancements to `cylc lint`: ### Fixes +[#5616](https://github.com/cylc/cylc-flow/pull/5616) - +Improve PBS support for job IDs with trailing components. + [#5619](https://github.com/cylc/cylc-flow/pull/5619) - Fix an issue where the `task_pool` table in the database wasn't being updated in a timely fashion when tasks completed. diff --git a/cylc/flow/job_runner_handlers/pbs.py b/cylc/flow/job_runner_handlers/pbs.py index c28ce3899ff..aa264311fc4 100644 --- a/cylc/flow/job_runner_handlers/pbs.py +++ b/cylc/flow/job_runner_handlers/pbs.py @@ -82,7 +82,7 @@ class PBSHandler: # system, so there is no need to filter its output. POLL_CMD = "qstat" POLL_CANT_CONNECT_ERR = "Connection refused" - REC_ID_FROM_SUBMIT_OUT = re.compile(r"""\A\s*(?P\S+)\s*\Z""") + REC_ID_FROM_SUBMIT_OUT = re.compile(r"^\s*(?P\d+)", re.M) SUBMIT_CMD_TMPL = "qsub '%(job)s'" def format_directives(self, job_conf): @@ -123,5 +123,10 @@ def format_directives(self, job_conf): lines.append(self.DIRECTIVE_PREFIX + key) return lines + @classmethod + def filter_poll_many_output(cls, out): + """Strip trailing stuff from the job ID.""" + return cls.REC_ID_FROM_SUBMIT_OUT.findall(out) + JOB_RUNNER_HANDLER = PBSHandler() diff --git a/cylc/flow/job_runner_mgr.py b/cylc/flow/job_runner_mgr.py index 60c68e7875c..f23c6c70104 100644 --- a/cylc/flow/job_runner_mgr.py +++ b/cylc/flow/job_runner_mgr.py @@ -403,7 +403,8 @@ def _create_nn(cls, job_file_path): rmtree( os.path.join(task_log_dir, name), ignore_errors=True) - def _filter_submit_output(self, st_file_path, job_runner, out, err): + @classmethod + def _filter_submit_output(cls, st_file_path, job_runner, out, err): """Filter submit command output, if relevant.""" job_id = None if hasattr(job_runner, "REC_ID_FROM_SUBMIT_ERR"): @@ -421,9 +422,9 @@ def _filter_submit_output(self, st_file_path, job_runner, out, err): job_id = job_runner.manip_job_id(job_id) with open(st_file_path, "a") as job_status_file: job_status_file.write("{0}={1}\n".format( - self.CYLC_JOB_ID, job_id)) + cls.CYLC_JOB_ID, job_id)) job_status_file.write("{0}={1}\n".format( - self.CYLC_JOB_RUNNER_SUBMIT_TIME, + cls.CYLC_JOB_RUNNER_SUBMIT_TIME, get_current_time_string())) break if hasattr(job_runner, "filter_submit_output"): diff --git a/tests/unit/job_runner_handlers/test_pbs.py b/tests/unit/job_runner_handlers/test_pbs.py index ac7c884046b..9bccf6a2314 100644 --- a/tests/unit/job_runner_handlers/test_pbs.py +++ b/tests/unit/job_runner_handlers/test_pbs.py @@ -20,6 +20,7 @@ JOB_RUNNER_HANDLER, PBSHandler ) +from cylc.flow.job_runner_mgr import JobRunnerManager VERY_LONG_STR = 'x' * 240 @@ -118,3 +119,43 @@ ) def test_format_directives(job_conf: dict, lines: list): assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines + + +def test_filter_poll_many_output(): + """It should strip trailing junk from job IDs. + + Job IDs are assumed to be a series of numbers, optionally followed by a + full-stop and some other letters and numbers which are not needed for + job tracking purposes. + + Job IDs are not expected to start with letters e.g. `abc.456` is not + supported. + """ + assert JOB_RUNNER_HANDLER.filter_poll_many_output(''' +Job id Name User Time Use S Queue +---------------- ---------------- ---------------- -------- - ----- +12345.foo.bar.baz test-pbs xxxxxxx 0 Q reomq +23456.foo test-pbs xxxxxxx 0 Q romeq +34567 test-pbs xxxxxxx 1 Q romeq +abc.456 test-pbs xxxxxxx 2 Q romeq +abcdef test-pbs xxxxxxx 2 Q romeq + ''') == ['12345', '23456', '34567'] + + + +def test_filter_submit_output(tmp_path): + """See notes for test_filter_poll_many_output.""" + status_file = tmp_path / 'submit_out' + status_file.touch() + + def test(out): + return JobRunnerManager._filter_submit_output( + status_file, + JOB_RUNNER_HANDLER, + out, + '', + )[2] + + assert test(' 12345.foo.bar.baz') == '12345' + assert test(' 12345.foo') == '12345' + assert test(' 12345') == '12345'