Merge pull request #5616 from oliver-sanders/4051

pbs: trim trailing text from job ids

CHANGES.md

@@ -77,6 +77,9 @@ Various enhancements to `cylc lint`:
 
 ### Fixes
 
+[#5616](https://github.com/cylc/cylc-flow/pull/5616) -
+Improve PBS support for job IDs with trailing components.
+
 [#5619](https://github.com/cylc/cylc-flow/pull/5619) -
 Fix an issue where the `task_pool` table in the database wasn't being updated
 in a timely fashion when tasks completed.

cylc/flow/job_runner_handlers/pbs.py

@@ -82,7 +82,7 @@ class PBSHandler:
     # system, so there is no need to filter its output.
     POLL_CMD = "qstat"
     POLL_CANT_CONNECT_ERR = "Connection refused"
-    REC_ID_FROM_SUBMIT_OUT = re.compile(r"""\A\s*(?P<id>\S+)\s*\Z""")
+    REC_ID_FROM_SUBMIT_OUT = re.compile(r"^\s*(?P<id>\d+)", re.M)
     SUBMIT_CMD_TMPL = "qsub '%(job)s'"
 
     def format_directives(self, job_conf):
@@ -123,5 +123,10 @@ def format_directives(self, job_conf):
                 lines.append(self.DIRECTIVE_PREFIX + key)
         return lines
 
+    @classmethod
+    def filter_poll_many_output(cls, out):
+        """Strip trailing stuff from the job ID."""
+        return cls.REC_ID_FROM_SUBMIT_OUT.findall(out)
+
 
 JOB_RUNNER_HANDLER = PBSHandler()

cylc/flow/job_runner_mgr.py

@@ -403,7 +403,8 @@ def _create_nn(cls, job_file_path):
                     rmtree(
                         os.path.join(task_log_dir, name), ignore_errors=True)
 
-    def _filter_submit_output(self, st_file_path, job_runner, out, err):
+    @classmethod
+    def _filter_submit_output(cls, st_file_path, job_runner, out, err):
         """Filter submit command output, if relevant."""
         job_id = None
         if hasattr(job_runner, "REC_ID_FROM_SUBMIT_ERR"):
@@ -421,9 +422,9 @@ def _filter_submit_output(self, st_file_path, job_runner, out, err):
                         job_id = job_runner.manip_job_id(job_id)
                     with open(st_file_path, "a") as job_status_file:
                         job_status_file.write("{0}={1}\n".format(
-                            self.CYLC_JOB_ID, job_id))
+                            cls.CYLC_JOB_ID, job_id))
                         job_status_file.write("{0}={1}\n".format(
-                            self.CYLC_JOB_RUNNER_SUBMIT_TIME,
+                            cls.CYLC_JOB_RUNNER_SUBMIT_TIME,
                             get_current_time_string()))
                     break
         if hasattr(job_runner, "filter_submit_output"):

tests/unit/job_runner_handlers/test_pbs.py

@@ -20,6 +20,7 @@
     JOB_RUNNER_HANDLER,
     PBSHandler
 )
+from cylc.flow.job_runner_mgr import JobRunnerManager
 
 
 VERY_LONG_STR = 'x' * 240
@@ -118,3 +119,43 @@
 )
 def test_format_directives(job_conf: dict, lines: list):
     assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines
+
+
+def test_filter_poll_many_output():
+    """It should strip trailing junk from job IDs.
+
+    Job IDs are assumed to be a series of numbers, optionally followed by a
+    full-stop and some other letters and numbers which are not needed for
+    job tracking purposes.
+
+    Job IDs are not expected to start with letters e.g. `abc.456` is not
+    supported.
+    """
+    assert JOB_RUNNER_HANDLER.filter_poll_many_output('''
+Job id            Name             User              Time Use S Queue
+----------------  ---------------- ----------------  -------- - -----
+12345.foo.bar.baz test-pbs         xxxxxxx                  0 Q reomq
+23456.foo         test-pbs         xxxxxxx                  0 Q romeq
+34567             test-pbs         xxxxxxx                  1 Q romeq
+abc.456           test-pbs         xxxxxxx                  2 Q romeq
+abcdef            test-pbs         xxxxxxx                  2 Q romeq
+    ''') == ['12345', '23456', '34567']
+
+
+
+def test_filter_submit_output(tmp_path):
+    """See notes for test_filter_poll_many_output."""
+    status_file = tmp_path / 'submit_out'
+    status_file.touch()
+
+    def test(out):
+        return JobRunnerManager._filter_submit_output(
+            status_file,
+            JOB_RUNNER_HANDLER,
+            out,
+            '',
+        )[2]
+
+    assert test('   12345.foo.bar.baz') == '12345'
+    assert test('   12345.foo') == '12345'
+    assert test('   12345') == '12345'