diff --git a/scuba/config.py b/scuba/config.py index 0cb5e24..c06d788 100644 --- a/scuba/config.py +++ b/scuba/config.py @@ -30,6 +30,10 @@ class ConfigNotFoundError(ConfigError): pass +class Reference(list): + pass + + class OverrideMixin: """ A mixin class that indicates an instance's value should override something @@ -125,6 +129,9 @@ def from_yaml(self, node: yaml.nodes.Node) -> Any: ] # Be sure to replace any escaped '.' characters with *just* the '.' except KeyError: raise yaml.YAMLError(f"Key {key!r} not found in {filename}") + + if isinstance(cur, Reference): + cur = self._process_reference(doc, cur) return cur def override(self, node: yaml.nodes.Node) -> OverrideMixin: @@ -155,127 +162,52 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin: assert isinstance(obj, OverrideMixin) return obj - def from_gitlab(self, node: yaml.nodes.Node) -> Any: - """ - Implementes a !from_gitlab constructor with the following syntax: - !from_gitlab filename key. - - Similar to !from_yaml, with extended .gitlab-ci.yml support. - - Arguments: - filename: Filename of external YAML document from which to load, - relative to the current YAML file. - key: Key from external YAML document to return, - using a dot-separated syntax for nested keys. - - Examples: - !from_gitlab external.yml pop - !from_gitlab external.yml foo.bar.pop - !from_gitlab "another file.yml" "foo bar.snap crackle.pop" - """ - - # Load the content from the node, as a scalar - assert isinstance(node, yaml.nodes.ScalarNode) - content = self.construct_scalar(node) - assert isinstance(content, str) - - # Split on unquoted spaces - parts = shlex.split(content) - if len(parts) != 2: - raise yaml.YAMLError("Two arguments expected to !from_yaml") - filename, key = parts - - # path is relative to the current YAML document - path = self._root / filename - - # Load the other YAML document - doc = self._cache.get(path) - if not doc: - with path.open("r") as f: - doc = yaml.load(f, GitlabYamlLoader) - self._cache[path] = doc - - # Retrieve the key - try: - cur = doc - # Use a negative look-behind to split the key on non-escaped '.' characters - for k in re.split(r"(? document - # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8 - def reference(self, node:yaml.nodes.Node): + def reference(self, node: yaml.nodes.Node) -> Reference: """ Implementes a !reference constructor with the following syntax: - !reference [comma, separated, ] - - Arguments: - filename: Filename of external YAML document from which to load, - relative to the current YAML file. - key: Key from external YAML document to return, - using a dot-separated syntax for nested keys. + !reference [comma, separated, path] Examples: - !from_yaml external.yml pop - !from_yaml external.yml foo.bar.pop - !from_yaml "another file.yml" "foo bar.snap crackle.pop" + !reference [job, image] + !reference [other_job, before_script] + !reference [last_job, variables] """ + # Load the content from the node, as a sequence + assert isinstance(node, yaml.nodes.SequenceNode) + key = self.construct_sequence(node) + assert isinstance(key, list) + return Reference(key) - # Load the content from the node, as a scalar - assert isinstance(node, yaml.nodes.ScalarNode) - content = self.construct_scalar(node) - assert isinstance(content, str) - - # Split on unquoted spaces - parts = shlex.split(content) - if len(parts) != 2: - raise yaml.YAMLError("Two arguments expected to !from_yaml") - filename, key = parts - - # path is relative to the current YAML document - path = self._root / filename - - # Load the other YAML document - doc = self._cache.get(path) - if not doc: - with path.open("r") as f: - doc = yaml.load(f, self.__class__) - self._cache[path] = doc - + def _process_reference(self, doc: dict, key: Reference) -> Any: + """ + Converts a reference (list of yaml keys) to its referenced value + """ # Retrieve the key try: cur = doc - # Use a negative look-behind to split the key on non-escaped '.' characters - for k in re.split(r"(? Any: + def extends(self, node: yaml.nodes.Node) -> Any: pass - def includes(self, node:yaml.nodes.Node) -> Any: + + def includes(self, node: yaml.nodes.Node) -> Any: pass -GitlabYamlLoader.add_constructor("!reference", GitlabYamlLoader.reference) -GitlabYamlLoader.add_constructor("includes", GitlabYamlLoader.includes) -GitlabYamlLoader.add_constructor("extends", GitlabYamlLoader.extends) + +Loader.add_constructor("!from_yaml", Loader.from_yaml) +Loader.add_constructor("!override", Loader.override) +# Gitlab specific extensions +Loader.add_constructor("!reference", Loader.reference) +Loader.add_constructor("includes", Loader.includes) +Loader.add_constructor("extends", Loader.extends) + def find_config() -> Tuple[Path, Path, ScubaConfig]: """Search up the directory hierarchy for .scuba.yml @@ -446,13 +378,11 @@ def _get_typed_val( @overload # When default is None, can return None (Optional). -def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]: - ... +def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]: ... @overload # When default is non-None, cannot return None. -def _get_str(data: CfgData, key: str, default: str) -> str: - ... +def _get_str(data: CfgData, key: str, default: str) -> str: ... def _get_str(data: CfgData, key: str, default: Optional[str] = None) -> Optional[str]: diff --git a/tests/test_config.py b/tests/test_config.py index 95b255e..877cff3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -282,6 +282,32 @@ def test_load_config_safe_external(self) -> None: SCUBA_YML.write_text(f"image: !from_yaml {external_yml} danger") self.__test_load_config_safe(external_yml) + def test_load_config_from_gitlab_with_reference(self) -> None: + """load_config loads a config using !from_gitlab with !reference tag""" + GITLAB_YML.write_text( + """ + .start: + here: dummian:8.2 + + now: + here: !reference [.start, here] + """ + ) + config = load_config(config_text=f'image: !from_yaml {GITLAB_YML} "now.here"\n') + assert config.image == "dummian:8.2" + + def test_load_config_from_gitlab_with_bad_reference(self) -> None: + """load_config loads a config using !from_gitlab with !reference tag""" + GITLAB_YML.write_text( + """ + now: + here: !reference [.start, here] + """ + ) + invalid_config( + config_text=f'image: !from_yaml {GITLAB_YML} "now.here"\n', + ) + class TestConfigHooks(ConfigTest): def test_hooks_mixed(self) -> None: