Skip to content

Commit

Permalink
pbs: trim trailing text from job ids
Browse files Browse the repository at this point in the history
* Closes #4051
  • Loading branch information
oliver-sanders committed Jul 10, 2023
1 parent 125c443 commit 7e96003
Show file tree
Hide file tree
Showing 3 changed files with 51 additions and 4 deletions.
7 changes: 6 additions & 1 deletion cylc/flow/job_runner_handlers/pbs.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class PBSHandler:
# system, so there is no need to filter its output.
POLL_CMD = "qstat"
POLL_CANT_CONNECT_ERR = "Connection refused"
REC_ID_FROM_SUBMIT_OUT = re.compile(r"""\A\s*(?P<id>\S+)\s*\Z""")
REC_ID_FROM_SUBMIT_OUT = re.compile(r"^\s*(?P<id>\d+)", re.M)
SUBMIT_CMD_TMPL = "qsub '%(job)s'"

def format_directives(self, job_conf):
Expand Down Expand Up @@ -123,5 +123,10 @@ def format_directives(self, job_conf):
lines.append(self.DIRECTIVE_PREFIX + key)
return lines

@classmethod
def filter_poll_many_output(cls, out):
"""Strip trailing stuff from the job ID."""
return cls.REC_ID_FROM_SUBMIT_OUT.findall(out)


JOB_RUNNER_HANDLER = PBSHandler()
7 changes: 4 additions & 3 deletions cylc/flow/job_runner_mgr.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,8 @@ def _create_nn(cls, job_file_path):
rmtree(
os.path.join(task_log_dir, name), ignore_errors=True)

def _filter_submit_output(self, st_file_path, job_runner, out, err):
@classmethod
def _filter_submit_output(cls, st_file_path, job_runner, out, err):
"""Filter submit command output, if relevant."""
job_id = None
if hasattr(job_runner, "REC_ID_FROM_SUBMIT_ERR"):
Expand All @@ -421,9 +422,9 @@ def _filter_submit_output(self, st_file_path, job_runner, out, err):
job_id = job_runner.manip_job_id(job_id)
with open(st_file_path, "a") as job_status_file:
job_status_file.write("{0}={1}\n".format(
self.CYLC_JOB_ID, job_id))
cls.CYLC_JOB_ID, job_id))
job_status_file.write("{0}={1}\n".format(
self.CYLC_JOB_RUNNER_SUBMIT_TIME,
cls.CYLC_JOB_RUNNER_SUBMIT_TIME,
get_current_time_string()))
break
if hasattr(job_runner, "filter_submit_output"):
Expand Down
41 changes: 41 additions & 0 deletions tests/unit/job_runner_handlers/test_pbs.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
JOB_RUNNER_HANDLER,
PBSHandler
)
from cylc.flow.job_runner_mgr import JobRunnerManager


VERY_LONG_STR = 'x' * 240
Expand Down Expand Up @@ -118,3 +119,43 @@
)
def test_format_directives(job_conf: dict, lines: list):
assert JOB_RUNNER_HANDLER.format_directives(job_conf) == lines


def test_filter_poll_many_output():
"""It should strip trailing junk from job IDs.
Job IDs are assumed to be a series of numbers, optionally followed by a
full-stop and some other letters and numbers which are not needed for
job tracking purposes.
Job IDs are not expected to start with letters e.g. `abc.456` is not
supported.
"""
assert JOB_RUNNER_HANDLER.filter_poll_many_output('''
Job id Name User Time Use S Queue
---------------- ---------------- ---------------- -------- - -----
12345.foo.bar.baz test-pbs xxxxxxx 0 Q reomq
23456.foo test-pbs xxxxxxx 0 Q romeq
34567 test-pbs xxxxxxx 1 Q romeq
abc.456 test-pbs xxxxxxx 2 Q romeq
abcdef test-pbs xxxxxxx 2 Q romeq
''') == ['12345', '23456', '34567']



def test_filter_submit_output(tmp_path):
"""See notes for test_filter_poll_many_output."""
status_file = tmp_path / 'submit_out'
status_file.touch()

def test(out):
return JobRunnerManager._filter_submit_output(
status_file,
JOB_RUNNER_HANDLER,
out,
'',
)[2]

assert test(' 12345.foo.bar.baz') == '12345'
assert test(' 12345.foo') == '12345'
assert test(' 12345') == '12345'

0 comments on commit 7e96003

Please sign in to comment.