Skip to content

Commit

Permalink
add !reference within same file
Browse files Browse the repository at this point in the history
  • Loading branch information
funkecoder23 authored and funkecoder23 committed Jun 27, 2024
1 parent b1fba88 commit 7210838
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 107 deletions.
144 changes: 37 additions & 107 deletions scuba/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ class ConfigNotFoundError(ConfigError):
pass


class Reference(list):
pass


class OverrideMixin:
"""
A mixin class that indicates an instance's value should override something
Expand Down Expand Up @@ -125,6 +129,9 @@ def from_yaml(self, node: yaml.nodes.Node) -> Any:
] # Be sure to replace any escaped '.' characters with *just* the '.'
except KeyError:
raise yaml.YAMLError(f"Key {key!r} not found in {filename}")

if isinstance(cur, Reference):
cur = self._process_reference(doc, cur)
return cur

def override(self, node: yaml.nodes.Node) -> OverrideMixin:
Expand Down Expand Up @@ -155,127 +162,52 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin:
assert isinstance(obj, OverrideMixin)
return obj

def from_gitlab(self, node: yaml.nodes.Node) -> Any:
"""
Implementes a !from_gitlab constructor with the following syntax:
!from_gitlab filename key.
Similar to !from_yaml, with extended .gitlab-ci.yml support.
Arguments:
filename: Filename of external YAML document from which to load,
relative to the current YAML file.
key: Key from external YAML document to return,
using a dot-separated syntax for nested keys.
Examples:
!from_gitlab external.yml pop
!from_gitlab external.yml foo.bar.pop
!from_gitlab "another file.yml" "foo bar.snap crackle.pop"
"""

# Load the content from the node, as a scalar
assert isinstance(node, yaml.nodes.ScalarNode)
content = self.construct_scalar(node)
assert isinstance(content, str)

# Split on unquoted spaces
parts = shlex.split(content)
if len(parts) != 2:
raise yaml.YAMLError("Two arguments expected to !from_yaml")
filename, key = parts

# path is relative to the current YAML document
path = self._root / filename

# Load the other YAML document
doc = self._cache.get(path)
if not doc:
with path.open("r") as f:
doc = yaml.load(f, GitlabYamlLoader)
self._cache[path] = doc

# Retrieve the key
try:
cur = doc
# Use a negative look-behind to split the key on non-escaped '.' characters
for k in re.split(r"(?<!\\)\.", key):
cur = cur[
k.replace("\\.", ".")
] # Be sure to replace any escaped '.' characters with *just* the '.'
except KeyError:
raise yaml.YAMLError(f"Key {key!r} not found in {filename}")
return cur

Loader.add_constructor("!from_gitlab", Loader.from_gitlab)
Loader.add_constructor("!from_yaml", Loader.from_yaml)
Loader.add_constructor("!override", Loader.override)

class GitlabYamlLoader(yaml.SafeLoader):
_root: Path # directory containing the loaded document
_cache: Dict[Path, Any] # document path => document

# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
# https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8
def reference(self, node:yaml.nodes.Node):
def reference(self, node: yaml.nodes.Node) -> Reference:
"""
Implementes a !reference constructor with the following syntax:
!reference [comma, separated, ]
Arguments:
filename: Filename of external YAML document from which to load,
relative to the current YAML file.
key: Key from external YAML document to return,
using a dot-separated syntax for nested keys.
!reference [comma, separated, path]
Examples:
!from_yaml external.yml pop
!from_yaml external.yml foo.bar.pop
!from_yaml "another file.yml" "foo bar.snap crackle.pop"
!reference [job, image]
!reference [other_job, before_script]
!reference [last_job, variables]
"""
# Load the content from the node, as a sequence
assert isinstance(node, yaml.nodes.SequenceNode)
key = self.construct_sequence(node)
assert isinstance(key, list)
return Reference(key)

# Load the content from the node, as a scalar
assert isinstance(node, yaml.nodes.ScalarNode)
content = self.construct_scalar(node)
assert isinstance(content, str)

# Split on unquoted spaces
parts = shlex.split(content)
if len(parts) != 2:
raise yaml.YAMLError("Two arguments expected to !from_yaml")
filename, key = parts

# path is relative to the current YAML document
path = self._root / filename

# Load the other YAML document
doc = self._cache.get(path)
if not doc:
with path.open("r") as f:
doc = yaml.load(f, self.__class__)
self._cache[path] = doc

def _process_reference(self, doc: dict, key: Reference) -> Any:
"""
Converts a reference (list of yaml keys) to its referenced value
"""
# Retrieve the key
try:
cur = doc
# Use a negative look-behind to split the key on non-escaped '.' characters
for k in re.split(r"(?<!\\)\.", key):
cur = cur[
k.replace("\\.", ".")
] # Be sure to replace any escaped '.' characters with *just* the '.'
for k in key:
cur = cur[k]
except KeyError:
raise yaml.YAMLError(f"Key {key!r} not found in {filename}")
raise yaml.YAMLError(f"Key {key!r} not found")
return cur

# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
def extends(self, node:yaml.nodes.Node) -> Any:
def extends(self, node: yaml.nodes.Node) -> Any:
pass
def includes(self, node:yaml.nodes.Node) -> Any:

def includes(self, node: yaml.nodes.Node) -> Any:
pass

GitlabYamlLoader.add_constructor("!reference", GitlabYamlLoader.reference)
GitlabYamlLoader.add_constructor("includes", GitlabYamlLoader.includes)
GitlabYamlLoader.add_constructor("extends", GitlabYamlLoader.extends)

Loader.add_constructor("!from_yaml", Loader.from_yaml)
Loader.add_constructor("!override", Loader.override)
# Gitlab specific extensions
Loader.add_constructor("!reference", Loader.reference)
Loader.add_constructor("includes", Loader.includes)
Loader.add_constructor("extends", Loader.extends)


def find_config() -> Tuple[Path, Path, ScubaConfig]:
"""Search up the directory hierarchy for .scuba.yml
Expand Down Expand Up @@ -446,13 +378,11 @@ def _get_typed_val(


@overload # When default is None, can return None (Optional).
def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]:
...
def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]: ...


@overload # When default is non-None, cannot return None.
def _get_str(data: CfgData, key: str, default: str) -> str:
...
def _get_str(data: CfgData, key: str, default: str) -> str: ...


def _get_str(data: CfgData, key: str, default: Optional[str] = None) -> Optional[str]:
Expand Down
26 changes: 26 additions & 0 deletions tests/test_config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,32 @@ def test_load_config_safe_external(self) -> None:
SCUBA_YML.write_text(f"image: !from_yaml {external_yml} danger")
self.__test_load_config_safe(external_yml)

def test_load_config_from_gitlab_with_reference(self) -> None:
"""load_config loads a config using !from_gitlab with !reference tag"""
GITLAB_YML.write_text(
"""
.start:
here: dummian:8.2
now:
here: !reference [.start, here]
"""
)
config = load_config(config_text=f'image: !from_yaml {GITLAB_YML} "now.here"\n')
assert config.image == "dummian:8.2"

def test_load_config_from_gitlab_with_bad_reference(self) -> None:
"""load_config loads a config using !from_gitlab with !reference tag"""
GITLAB_YML.write_text(
"""
now:
here: !reference [.start, here]
"""
)
invalid_config(
config_text=f'image: !from_yaml {GITLAB_YML} "now.here"\n',
)


class TestConfigHooks(ConfigTest):
def test_hooks_mixed(self) -> None:
Expand Down

0 comments on commit 7210838

Please sign in to comment.