add support for reference in scripts

scuba/config.py

@@ -55,12 +55,15 @@ class OverrideList(list, OverrideMixin):
 class OverrideStr(str, OverrideMixin):
     pass
 
+
 class Reference(list):
     """
     Represents a `!reference` tag value that needs to be parsed after yaml is loaded
     """
+
     pass
 
+
 # http://stackoverflow.com/a/9577670
 class Loader(yaml.SafeLoader):
     _root: Path  # directory containing the loaded document
@@ -181,9 +184,19 @@ def from_gitlab(self, node: yaml.nodes.Node) -> Any:
                 ]  # Be sure to replace any escaped '.' characters with *just* the '.'
         except KeyError:
             raise yaml.YAMLError(f"Key {key!r} not found in {filename}")
-        
+
         if isinstance(cur, Reference):
             cur = _process_reference(doc, cur)
+        if isinstance(cur, list):
+            new_cur = []
+            for c in cur:
+                if isinstance(c, Reference):
+                    # use += to concatenate list type
+                    new_cur += _process_reference(doc, c)
+                else:
+                    # use append to concatenate other types
+                    new_cur.append(c)
+            cur = new_cur
 
         return cur
 
@@ -220,10 +233,11 @@ def override(self, node: yaml.nodes.Node) -> OverrideMixin:
 Loader.add_constructor("!override", Loader.override)
 Loader.add_constructor("!from_gitlab", Loader.from_gitlab)
 
+
 class GitlabLoader(Loader):
     # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
     # https://gitlab.com/gitlab-org/gitlab/-/blob/436d642725ac6675c97c7e5833d8427e8422ac78/lib/gitlab/ci/config/yaml/tags/reference.rb#L8
-    
+
     def reference(self, node: yaml.nodes.Node) -> Reference:
         """
         Implements a !reference constructor with the following syntax:
@@ -242,24 +256,10 @@ def reference(self, node: yaml.nodes.Node) -> Reference:
         assert isinstance(key, list)
         return Reference(key)
 
-
-def _process_reference(doc: dict, key: Reference) -> Any:
-    """
-    Converts a reference (list of yaml keys) to its referenced value
-    """
-    # Retrieve the key
-    try:
-        cur = doc
-        for k in key:
-            cur = cur[k]
-    except KeyError:
-        raise yaml.YAMLError(f"Key {key!r} not found")
-    return cur
 
 GitlabLoader.add_constructor("!reference", GitlabLoader.reference)
 
 
-
 def find_config() -> Tuple[Path, Path, ScubaConfig]:
     """Search up the directory hierarchy for .scuba.yml
 
@@ -333,6 +333,9 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]:
         # The script is just the text itself
         return [node]
 
+    if isinstance(node, list):
+        return node
+
     if isinstance(node, dict):
         # There must be a "script" key, which must be a list of strings
         script = node.get("script")
@@ -350,6 +353,26 @@ def _process_script_node(node: CfgNode, name: str) -> List[str]:
     raise ConfigError(f"{name}: must be string or dict")
 
 
+def _process_reference(doc: dict, key: Reference) -> Any:
+    """Process a reference tag
+
+    Args:
+      doc: a yaml document
+      key: the reference to be parsed
+
+    Returns:
+      the referenced value
+    """
+    # Retrieve the key
+    try:
+        cur = doc
+        for k in key:
+            cur = cur[k]
+    except KeyError:
+        raise yaml.YAMLError(f"Key {key!r} not found")
+    return cur
+
+
 def _process_environment(node: CfgNode, name: str) -> Environment:
     # Environment can be either a list of strings ("KEY=VALUE") or a mapping
     # Environment keys and values are always strings
@@ -429,13 +452,11 @@ def _get_typed_val(
 
 
 @overload  # When default is None, can return None (Optional).
-def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]:
-    ...
+def _get_str(data: CfgData, key: str, default: None = None) -> Optional[str]: ...
 
 
 @overload  # When default is non-None, cannot return None.
-def _get_str(data: CfgData, key: str, default: str) -> str:
-    ...
+def _get_str(data: CfgData, key: str, default: str) -> str: ...
 
 
 def _get_str(data: CfgData, key: str, default: Optional[str] = None) -> Optional[str]:

tests/test_config.py

@@ -298,6 +298,57 @@ def test_load_config_from_gitlab_with_reference(self) -> None:
         )
         assert config.image == "dummian:8.2"
 
+    def test_load_config_from_gitlab_reference_script_str(self) -> None:
+        """load a .gitlab-ci.yml with !reference in script"""
+        GITLAB_YML.write_text(
+            """
+            .setup:
+              script:
+                - do-something-really-important
+                - and-another-thing
+
+            build:
+              stage: build
+              script: !reference [.setup, script]
+            """
+        )
+        config = load_config(
+            config_text=f"""
+            aliases:
+              important: !from_gitlab {GITLAB_YML} build.script
+            """
+        )
+        assert config.aliases["important"].script == [
+            "do-something-really-important",
+            "and-another-thing",
+        ]
+
+    def test_load_config_from_gitlab_reference_script_list(self) -> None:
+        """load a .gitlab-ci.yml with !reference in script"""
+        GITLAB_YML.write_text(
+            """
+            .setup:
+              script:
+                - do-something-really-important
+
+            build:
+              stage: build
+              script:
+                - !reference [.setup, script]
+                - depends-on-important-stuff
+            """
+        )
+        config = load_config(
+            config_text=f"""
+            image: bosybux
+            aliases:
+              important: !from_gitlab {GITLAB_YML} build.script
+            """
+        )
+        assert config.image == "bosybux"
+        assert len(config.aliases) == 1
+        assert config.aliases["important"].script == ["do-something-really-important", "depends-on-important-stuff"]
+
     def test_load_config_from_gitlab_with_bad_reference(self) -> None:
         """load_config loads a config using !from_gitlab with !reference tag"""
         GITLAB_YML.write_text(