Fix cylc clean while cat-log running

cylc/flow/cfgspec/globalcfg.py

@@ -1426,7 +1426,8 @@ def default_for(
                        "[remote]retrieve job logs retry delays")}
             ''')
             Conf('tail command template',
-                 VDR.V_STRING, 'tail -n +1 -F %(filename)s', desc=f'''
+                 VDR.V_STRING, 'tail -n +1 --follow=name -F %(filename)s',
+                 desc=f'''
                 A command template (with ``%(filename)s`` substitution) to
                 tail-follow job logs this platform, by ``cylc cat-log``.
 

cylc/flow/pathutil.py

@@ -15,16 +15,18 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Functions to return paths to common workflow files and directories."""
 
+import errno
 import os
 from pathlib import Path
 import re
 from shutil import rmtree
+from time import sleep
 from typing import Dict, Iterable, Set, Union, Optional, Any
 
 from cylc.flow import LOG
 from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.exceptions import (
-    InputError, WorkflowFilesError, handle_rmtree_err
+    FileRemovalError, InputError, WorkflowFilesError
 )
 from cylc.flow.platforms import get_localhost_install_target
 
@@ -297,15 +299,35 @@ def remove_dir_and_target(path: Union[Path, str]) -> None:
                 "Removing symlink and its target directory: "
                 f"{path} -> {target}"
             )
-            rmtree(target, onerror=handle_rmtree_err)
+            _rmtree(target)
         else:
             LOG.info(f'Removing broken symlink: {path}')
         os.remove(path)
     elif not os.path.exists(path):
         raise FileNotFoundError(path)
     else:
         LOG.info(f'Removing directory: {path}')
-        rmtree(path, onerror=handle_rmtree_err)
+        _rmtree(path)
+
+
+def _rmtree(target: Union[Path, str],
+            retries: int = 5,
+            sleep_time: float = 0.5):
+    """Make rmtree more robust to nfs issues.
+
+    shutil.rmtree will be retried (default 5 times)
+    """
+    for _try_num in range(retries):
+        try:
+            rmtree(target)
+            return
+        except OSError as exc:
+            if exc.errno == errno.ENOTEMPTY:
+                err = exc
+                sleep(sleep_time)
+            else:
+                raise
+    raise FileRemovalError(err)
 
 
 def remove_dir_or_file(path: Union[Path, str]) -> None:
@@ -325,7 +347,7 @@ def remove_dir_or_file(path: Union[Path, str]) -> None:
         os.remove(path)
     else:
         LOG.info(f"Removing directory: {path}")
-        rmtree(path, onerror=handle_rmtree_err)
+        _rmtree(path)
 
 
 def remove_empty_parents(

cylc/flow/scripts/clean.py

@@ -193,15 +193,17 @@ async def run(*ids: str, opts: 'Values') -> None:
     if multi_mode and not opts.skip_interactive:
         prompt(workflows)  # prompt for approval or exit
 
-    failed = []
+    failed = {}
     for workflow in sorted(workflows):
         try:
             init_clean(workflow, opts)
         except Exception as exc:
-            failed.append(workflow)
-            LOG.warning(exc)
+            failed[workflow] = exc
     if failed:
-        raise CylcError(f"Clean failed: {', '.join(failed)}")
+        msg = "Clean failed:"
+        for workflow, exc_message in failed.items():
+            msg += f"\nWorkflow: {workflow}\nError: {exc_message}"
+        raise CylcError(msg)
 
 
 @cli_function(get_option_parser)