From 55538c5cff46b72b35866c6c1df857c5a256da9e Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Fri, 22 Apr 2022 17:58:05 +0100 Subject: [PATCH] jinja2: support zero-prefixed integers Jinja2v2 used to support zero-prefixed integers e.g. 01, 02, 03, ... Jinja2v3 dropped support for this, fair enough, however, our users haven't been getting warnings for this so haven't had time to adapt. This PR patches the Jinja2 parser we use in cylc-rose to shave off the leading zero before we pass the template variable back through to Jinja2 itself. It's not a particularly nice solution, it is intended as a stopgap to help users to migrate with the intention that we would retire it ASAP, probably with the move to Jinja2 3.1. --- CHANGES.md | 19 +++-- cylc/rose/jinja2_parser.py | 160 ++++++++++++++++++++++++++++++++++++- cylc/rose/utilities.py | 38 +++++---- 3 files changed, 192 insertions(+), 25 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 4ef0fbbe..f5cd9ffe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,21 +1,30 @@ # Selected Cylc-Rose Changes +## __cylc-rose-1.? (Pending)__ + +### Fixes + +[#140](https://github.com/cylc/cylc-rose/pull/140) - +Support integers with leading zeros (e.g `001`) to back support Rose +configurations for use with cylc-flow>=8.0rc5 which uses Jinja2 v3 which +no longer supports this. + ## __cylc-rose-1.0.3 (Released 2022-05-20)__ ### Fixes -[139](https://github.com/cylc/cylc-rose/pull/139) - Make `rose stem` command +[#139](https://github.com/cylc/cylc-rose/pull/139) - Make `rose stem` command work correctly with changes made to `cylc install` in [cylc-flow PR #4823](https://github.com/cylc/cylc-flow/pull/4823) -[130](https://github.com/cylc/cylc-rose/pull/130) - Fix bug preventing +[#130](https://github.com/cylc/cylc-rose/pull/130) - Fix bug preventing ``cylc reinstall`` using Rose fileinstall. -[132](https://github.com/cylc/cylc-rose/pull/132) - Fix bug preventing +[#132](https://github.com/cylc/cylc-rose/pull/132) - Fix bug preventing Cylc commands (other than `install`) from accessing the content of `--rose-template-variable`. -[133](https://github.com/cylc/cylc-rose/pull/133) - Fix bug allowing setting +[#133](https://github.com/cylc/cylc-rose/pull/133) - Fix bug allowing setting multiple template variable sections. ## __cylc-rose-1.0.2 (Released 2022-03-24)__ @@ -25,7 +34,7 @@ multiple template variable sections. [118](https://github.com/cylc/cylc-rose/pull/118) - Fail if a workflow is not a Rose Suite but user provides Rose CLI options. -## cylc-rose-1.0.1 (Released 2022-02-17) +## __cylc-rose-1.0.1 (Released 2022-02-17)__ First official release of Cylc-Rose. diff --git a/cylc/rose/jinja2_parser.py b/cylc/rose/jinja2_parser.py index 780e88cb..28716385 100644 --- a/cylc/rose/jinja2_parser.py +++ b/cylc/rose/jinja2_parser.py @@ -16,6 +16,8 @@ """Utility for parsing Jinja2 expressions.""" from ast import literal_eval as python_literal_eval +from copy import deepcopy +from contextlib import contextmanager import re from jinja2.nativetypes import NativeEnvironment # type: ignore @@ -27,6 +29,155 @@ Neg, Pos ) +import jinja2.lexer + +from cylc.flow import LOG + + +def _strip_leading_zeros(string): + """Strip leading zeros from a string. + + Examples: + >>> _strip_leading_zeros('1') + '1' + >>> _strip_leading_zeros('01') + '1' + >>> _strip_leading_zeros('001') + '1' + >>> _strip_leading_zeros('0001') + '1' + + """ + return string.lstrip('0') + + +def _lexer_wrap(fcn): + """Helper for patch_jinja2_leading_zeros. + + Patches the jinja2.lexer.Lexer.wrap method. + """ + instances = set() + + def _stream(stream): + """Patch the token stream to strip the leading zero where necessary.""" + nonlocal instances # record of uses of deprecated syntax + for lineno, token, value_str in stream: + if ( + token == jinja2.lexer.TOKEN_INTEGER + and len(value_str) > 1 + and value_str[0] == '0' + ): + instances.add(value_str) + yield (lineno, token, _strip_leading_zeros(value_str)) + + def _inner( + self, + stream, # : t.Iterable[t.Tuple[int, str, str]], + name, # : t.Optional[str] = None, + filename, # : t.Optional[str] = None, + ): # -> t.Iterator[Token]: + nonlocal fcn + return fcn(self, _stream(stream), name, filename) + + _inner.__wrapped__ = fcn # save the un-patched function + _inner._instances = instances # save the set of uses of deprecated syntax + + return _inner + + +@contextmanager +def patch_jinja2_leading_zeros(): + """Back support integers with leading zeros in Jinja2 v3. + + Jinja2 v3 dropped support for integers with leading zeros, these are + heavily used throughout Rose configurations. Since there was no deprecation + warning in Jinja2 v2 we have implemented this patch to extend support for + a short period to help our users to transition. + + This patch will issue a warning if usage of the deprecated syntax is + detected during the course of its usage. + + Warning: + This is a *global* patch applied to the Jinja2 library whilst the + context manager is open. Do not use this with parallel/async code + as the patch could apply to code outside of the context manager. + + Examples: + >>> env = NativeEnvironment() + + The integer "1" is ok: + >>> env.parse('{{ 1 }}') + Template(body=[Output(nodes=[Const(value=1)])]) + + However "01" is no longer supported: + >>> env.parse('{{ 01 }}') + Traceback (most recent call last): + jinja2.exceptions.TemplateSyntaxError: expected token ... + + The patch returns support (the leading-zero gets stripped): + >>> with patch_jinja2_leading_zeros(): + ... env.parse('{{ 01 }}') + Template(body=[Output(nodes=[Const(value=1)])]) + + The patch can handle any number of arbitrary leading zeros: + >>> with patch_jinja2_leading_zeros(): + ... env.parse('{{ 0000000001 }}') + Template(body=[Output(nodes=[Const(value=1)])]) + + Once the "with" closes we go back to vanilla Jinja2 behaviour: + >>> env.parse('{{ 01 }}') + Traceback (most recent call last): + jinja2.exceptions.TemplateSyntaxError: expected token ... + + """ + # clear any previously cashed lexer instances + jinja2.lexer._lexer_cache.clear() + + # apply the code patch (new lexer instances will pick up these changes) + _integer_re = deepcopy(jinja2.lexer.integer_re) + jinja2.lexer.integer_re = re.compile( + rf''' + # Jinja2 no longer recognises zero-padded integers as integers + # so we must patch its regex to allow them to be detected. + ( + [0-9](_?\d)* # decimal (which supports zero-padded integers) + | + {jinja2.lexer.integer_re.pattern} + ) + ''', + re.IGNORECASE | re.VERBOSE, + ) + jinja2.lexer.Lexer.wrap = _lexer_wrap(jinja2.lexer.Lexer.wrap) + + # execute the body of the "with" statement + yield + + # report any usage of deprecated syntax + if jinja2.lexer.Lexer.wrap._instances: + num_examples = 5 + LOG.warning( + 'Support for integers with leading zeros was dropped' + ' in Jinja2 v3.' + ' Rose will extend support until a future version.' + '\nPlease amend your Rose configuration files e.g:' + '\n * ' + + ( + '\n * '.join( + f'{before} => {_strip_leading_zeros(before)}' + for before in list( + jinja2.lexer.Lexer.wrap._instances + )[:num_examples] + ) + ) + + ) + + # revert the code patch + jinja2.lexer.integer_re = _integer_re + jinja2.lexer.Lexer.wrap = jinja2.lexer.Lexer.wrap.__wrapped__ + + # clear any patched lexers to return Jinja2 to normal operation + jinja2.lexer._lexer_cache.clear() class Parser(NativeEnvironment): @@ -80,13 +231,11 @@ def literal_eval(self, value): >>> parser.literal_eval('None') # valid jinja2 variants - >>> parser.literal_eval('042') - 42 >>> parser.literal_eval('true') True >>> parser.literal_eval('1,2,3') (1, 2, 3) - >>> parser.literal_eval('01,true,') + >>> parser.literal_eval('1,true,') (1, True) # multiline literals @@ -95,6 +244,11 @@ def literal_eval(self, value): >>> parser.literal_eval('1,\n2,\n3') (1, 2, 3) + # back-supported jinja2 variants + >>> with patch_jinja2_leading_zeros(): + ... parser.literal_eval('042') + 42 + # invalid examples >>> parser.literal_eval('1 + 1') Traceback (most recent call last): diff --git a/cylc/rose/utilities.py b/cylc/rose/utilities.py index 343b87a1..f4497ef5 100644 --- a/cylc/rose/utilities.py +++ b/cylc/rose/utilities.py @@ -25,7 +25,7 @@ from cylc.flow.hostuserutil import get_host from cylc.flow import LOG -from cylc.rose.jinja2_parser import Parser +from cylc.rose.jinja2_parser import Parser, patch_jinja2_leading_zeros from metomi.rose import __version__ as ROSE_VERSION from metomi.isodatetime.datetimeoper import DateTimeOperator from metomi.rose.config import ConfigDumper, ConfigNodeDiff, ConfigNode @@ -146,22 +146,26 @@ def get_rose_vars_from_config_node(config, config_node, environ): # Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic # access. if templating is not None: - parser = Parser() - for key, value in config['template_variables'].items(): - # The special variables are already Python variables. - if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']: - try: - config['template_variables'][key] = ( - parser.literal_eval(value) - ) - except Exception: - raise ConfigProcessError( - [templating, key], - value, - f'Invalid template variable: {value}' - '\nMust be a valid Python or Jinja2 literal' - ' (note strings "must be quoted").' - ) from None + with patch_jinja2_leading_zeros(): + # BACK COMPAT: patch_jinja2_leading_zeros + # back support zero-padded integers for a limited time to help + # users migrate before upgrading cylc-flow to Jinja2>=3.1 + parser = Parser() + for key, value in config['template_variables'].items(): + # The special variables are already Python variables. + if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']: + try: + config['template_variables'][key] = ( + parser.literal_eval(value) + ) + except Exception: + raise ConfigProcessError( + [templating, key], + value, + f'Invalid template variable: {value}' + '\nMust be a valid Python or Jinja2 literal' + ' (note strings "must be quoted").' + ) from None # Add ROSE_SUITE_VARIABLES to config of templating engines in use. if templating is not None: