From 9446fe5c347c2bc80313624156ddd01ea1677d1c Mon Sep 17 00:00:00 2001 From: funkecoder23 Date: Thu, 27 Jun 2024 13:55:11 -0400 Subject: [PATCH] add support for reference in scripts --- scuba/config.py | 71 +++++++++++++++++++++++++++++++------------- tests/test_config.py | 48 ++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 20 deletions(-) diff --git a/scuba/config.py b/scuba/config.py index 5804f60..c37ff83 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -55,12 +55,15 @@ class OverrideList(list, OverrideMixin): class OverrideStr(str, OverrideMixin): pass + class Reference(list): """ Represents a `!reference` tag value that needs to be parsed after yaml is loaded """ + pass + # http://stackoverflow.com/a/9577670 class Loader(yaml.SafeLoader): _root: Path # directory containing the loaded document @@ -181,9 +184,19 @@ def from_gitlab(self, node: yaml.nodes.Node) -> Any: ] # Be sure to replace any escaped '.' characters with *just* the '.' except KeyError: raise yaml.YAMLError(f"Key {key!r} not found in {filename}") - + if isinstance(cur, Reference): cur = _process_reference(doc, cur) + if isinstance(cur, list): + new_cur = [] + for c in cur: + if isinstance(c, Reference): + # use += to concatenate list type + new_cur += _process_reference(doc, c) + else: + # use append to concatenate other types + new_cur.append(c) + cur = new_cur return cur @@ -220,10 +233,11 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin: Loader.add_constructor("!override", Loader.override) Loader.add_constructor("!from_gitlab", Loader.from_gitlab) + class GitlabLoader(Loader): # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8 - + def reference(self, node: yaml.nodes.Node) -> Reference: """ Implementes a !reference constructor with the following syntax: @@ -242,24 +256,20 @@ def reference(self, node: yaml.nodes.Node) -> Reference: assert isinstance(key, list) return Reference(key) - -def _process_reference(doc: dict, key: Reference) -> Any: - """ - Converts a reference (list of yaml keys) to its referenced value - """ - # Retrieve the key - try: - cur = doc - for k in key: - cur = cur[k] - except KeyError: - raise yaml.YAMLError(f"Key {key!r} not found") - return cur + # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#use-extends-to-reuse-configuration-sections + def extends(self, node: yaml.nodes.Node) -> Any: + pass + + # https://docs.gitlab.com/ee/ci/yaml/index.html#include + def includes(self, node: yaml.nodes.Node) -> Any: + return self.from_gitlab(node) + # GitlabLoader.add_constructor("!from_gitlab", GitlabLoader.from_gitlab) GitlabLoader.add_constructor("!from_gitlab", GitlabLoader.from_gitlab) GitlabLoader.add_constructor("!reference", GitlabLoader.reference) - +GitlabLoader.add_constructor("includes", GitlabLoader.includes) +GitlabLoader.add_constructor("extends", GitlabLoader.extends) def find_config() -> Tuple[Path, Path, ScubaConfig]: @@ -335,6 +345,9 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: # The script is just the text itself return [node] + if isinstance(node, list): + return node + if isinstance(node, dict): # There must be a "script" key, which must be a list of strings script = node.get("script") @@ -352,6 +365,26 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]: raise ConfigError(f"{name}: must be string or dict") +def _process_reference(doc: dict, key: Reference) -> Any: + """Process a reference tag + + Args: + doc: a yaml document + key: the reference to be parsed + + Returns: + the referenced value + """ + # Retrieve the key + try: + cur = doc + for k in key: + cur = cur[k] + except KeyError: + raise yaml.YAMLError(f"Key {key!r} not found") + return cur + + def _process_environment(node: CfgNode, name: str) -> Environment: # Environment can be either a list of strings ("KEY=VALUE") or a mapping # Environment keys and values are always strings @@ -431,13 +464,11 @@ def _get_typed_val( @overload # When default is None, can return None (Optional). -def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]: - ... +def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]: ... @overload # When default is non-None, cannot return None. -def _get_str(data: CfgData, key: str, default: str) -> str: - ... +def _get_str(data: CfgData, key: str, default: str) -> str: ... def _get_str(data: CfgData, key: str, default: Optional[str] = None) -> Optional[str]: diff --git a/tests/test_config.py b/tests/test_config.py index e816da3..0c6666c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -298,6 +298,54 @@ def test_load_config_from_gitlab_with_reference(self) -> None: ) assert config.image == "dummian:8.2" + def test_load_config_from_gitlab_reference_script_str(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .setup: + script: + - do-something-really-important + - and-another-thing + + build: + stage: build + script: !reference [.setup, script] + """ + ) + config = load_config( + config_text=f""" + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.aliases["important"].script == ["do-something-really-important", "and-another-thing"] + + def test_load_config_from_gitlab_reference_script_list(self) -> None: + """load a .gitlab-ci.yml with !reference in script""" + GITLAB_YML.write_text( + """ + .setup: + script: + - do-something-really-important + + build: + stage: build + script: + - !reference [.setup, script] + - depends-on-important-stuff + """ + ) + config = load_config( + config_text=f""" + image: bosybux + aliases: + important: !from_gitlab {GITLAB_YML} build.script + """ + ) + assert config.image == "bosybux" + assert len(config.aliases) == 1 + assert config.aliases["important"].script == ["do-something-really-important", "depends-on-important-stuff"] + def test_load_config_from_gitlab_with_bad_reference(self) -> None: """load_config loads a config using !from_gitlab with !reference tag""" GITLAB_YML.write_text(