Add package argument to deep-merge in a JSON file during packaging

CHANGELOG.rst

@@ -7,12 +7,13 @@ Next Release (TBD)
 
 * Add experimental support for websockets
   (`#1017 <https://github.com/aws/chalice/issues/1017>`__)
-
 * API Gateway Endpoint Type Configuration
   (`#1160 https://github.com/aws/chalice/pull/1160`__)
-
 * API Gateway Resource Policy Configuration
   (`#1160 https://github.com/aws/chalice/pull/1160`__)
+* Add --merge-template option to package command
+  (`#1195 https://github.com/aws/chalice/pull/1195`__)
+
 
 1.9.1
 =====

chalice/cli/__init__.py

@@ -377,13 +377,20 @@ def generate_sdk(ctx, sdk_type, stage, outdir):
                     "this argument is specified, a single "
                     "zip file will be created instead."))
 @click.option('--stage', default=DEFAULT_STAGE_NAME)
+@click.option('--merge-template',
+              help=('Specify a JSON template to be merged '
+                    'into the generated template. This is useful '
+                    'for adding resources to a Chalice template or '
+                    'modify values in the template.'))
 @click.argument('out')
 @click.pass_context
-def package(ctx, single_file, stage, out):
-    # type: (click.Context, bool, str, str) -> None
+def package(ctx, single_file, stage, merge_template, out):
+    # type: (click.Context, bool, str, str, str) -> None
     factory = ctx.obj['factory']  # type: CLIFactory
-    config = factory.create_config_obj(stage)
-    packager = factory.create_app_packager(config)
+    config = factory.create_config_obj(
+        chalice_stage_name=stage,
+    )
+    packager = factory.create_app_packager(config, merge_template)
     if single_file:
         dirname = tempfile.mkdtemp()
         try:

chalice/cli/factory.py

@@ -138,13 +138,15 @@ def create_deployment_reporter(self, ui):
 
     def create_config_obj(self, chalice_stage_name=DEFAULT_STAGE_NAME,
                           autogen_policy=None,
-                          api_gateway_stage=None):
-        # type: (str, Optional[bool], str) -> Config
+                          api_gateway_stage=None,
+                          package_merge_template=None):
+        # type: (str, Optional[bool], str, Optional[str]) -> Config
         user_provided_params = {}  # type: Dict[str, Any]
         default_params = {'project_dir': self.project_dir,
                           'api_gateway_stage': DEFAULT_APIGATEWAY_STAGE_NAME,
                           'api_gateway_endpoint_type': DEFAULT_ENDPOINT_TYPE,
-                          'autogen_policy': True}
+                          'autogen_policy': True,
+                          'package_merge_template': None}
         try:
             config_from_disk = self.load_project_config()
         except (OSError, IOError):
@@ -161,6 +163,9 @@ def create_config_obj(self, chalice_stage_name=DEFAULT_STAGE_NAME,
             user_provided_params['profile'] = self.profile
         if api_gateway_stage is not None:
             user_provided_params['api_gateway_stage'] = api_gateway_stage
+        if package_merge_template is not None:
+            user_provided_params[
+                'package_merge_template'] = package_merge_template
         config = Config(chalice_stage=chalice_stage_name,
                         user_provided_params=user_provided_params,
                         config_from_disk=config_from_disk,
@@ -179,9 +184,9 @@ def _validate_config_from_disk(self, config):
         except ValueError:
             raise UnknownConfigFileVersion(string_version)
 
-    def create_app_packager(self, config):
-        # type: (Config) -> AppPackager
-        return create_app_packager(config)
+    def create_app_packager(self, config, merge_template=None):
+        # type: (Config, OptStr) -> AppPackager
+        return create_app_packager(config, merge_template=merge_template)
 
     def create_log_retriever(self, session, lambda_arn):
         # type: (Session, str) -> LogRetriever

chalice/package.py

@@ -1,7 +1,8 @@
 import os
 import copy
+import json
 
-from typing import Any, Dict, List, Set, Union  # noqa
+from typing import Any, Optional, Dict, List, Set, Union  # noqa
 from typing import cast
 
 from chalice.deploy.swagger import CFNSwaggerGenerator
@@ -14,8 +15,8 @@
 from chalice.deploy.deployer import create_build_stage
 
 
-def create_app_packager(config):
-    # type: (Config) -> AppPackager
+def create_app_packager(config, merge_template=None):
+    # type: (Config, Optional[str]) -> AppPackager
     osutils = OSUtils()
     ui = UI()
     application_builder = ApplicationGraphBuilder()
@@ -25,10 +26,18 @@ def create_app_packager(config):
     )
     resource_builder = ResourceBuilder(application_builder,
                                        deps_builder, build_stage)
+    processors = [
+        ReplaceCodeLocationPostProcessor(osutils=osutils),
+        TemplateMergePostProcessor(
+            osutils=osutils,
+            merger=TemplateDeepMerger(),
+            merge_template=merge_template,
+        ),
+    ]
     return AppPackager(
         SAMTemplateGenerator(),
         resource_builder,
-        TemplatePostProcessor(osutils=osutils),
+        CompositePostProcessor(processors),
         osutils,
     )
 
@@ -572,6 +581,12 @@ def __init__(self, osutils):
         # type: (OSUtils) -> None
         self._osutils = osutils
 
+    def process(self, template, config, outdir, chalice_stage_name):
+        # type: (Dict[str, Any], Config, str, str) -> None
+        raise NotImplementedError('process')
+
+
+class ReplaceCodeLocationPostProcessor(TemplatePostProcessor):
     def process(self, template, config, outdir, chalice_stage_name):
         # type: (Dict[str, Any], Config, str, str) -> None
         self._fixup_deployment_package(template, outdir)
@@ -594,3 +609,71 @@ def _fixup_deployment_package(self, template, outdir):
                 self._osutils.copy(original_location, new_location)
                 copied = True
             resource['Properties']['CodeUri'] = './deployment.zip'
+
+
+class TemplateMergePostProcessor(TemplatePostProcessor):
+    def __init__(self, osutils, merger, merge_template=None):
+        # type: (OSUtils, TemplateMerger, Optional[str]) -> None
+        super(TemplateMergePostProcessor, self).__init__(osutils)
+        self._merger = merger
+        self._merge_template = merge_template
+
+    def process(self, template, config, outdir, chalice_stage_name):
+        # type: (Dict[str, Any], Config, str, str) -> None
+        if self._merge_template is None:
+            return
+        loaded_template = self._load_template_to_merge()
+        merged = self._merger.merge(loaded_template, template)
+        template.clear()
+        template.update(merged)
+
+    def _load_template_to_merge(self):
+        # type: () -> Dict[str, Any]
+        template_name = cast(str, self._merge_template)
+        filepath = os.path.abspath(template_name)
+        if not self._osutils.file_exists(filepath):
+            raise RuntimeError('Cannot find template file: %s' % filepath)
+        template_data = self._osutils.get_file_contents(filepath, binary=False)
+        try:
+            loaded_template = json.loads(template_data)
+        except ValueError:
+            raise RuntimeError(
+                'Expected %s to be valid JSON template.' % filepath)
+        return loaded_template
+
+
+class CompositePostProcessor(TemplatePostProcessor):
+    def __init__(self, processors):
+        # type: (List[TemplatePostProcessor]) -> None
+        self._processors = processors
+
+    def process(self, template, config, outdir, chalice_stage_name):
+        # type: (Dict[str, Any], Config, str, str) -> None
+        for processor in self._processors:
+            processor.process(template, config, outdir, chalice_stage_name)
+
+
+class TemplateMerger(object):
+    def merge(self, file_template, chalice_template):
+        # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any]
+        raise NotImplementedError('merge')
+
+
+class TemplateDeepMerger(TemplateMerger):
+    def merge(self, file_template, chalice_template):
+        # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any]
+        return self._merge(file_template, chalice_template)
+
+    def _merge(self, file_template, chalice_template):
+        # type: (Any, Any) -> Any
+        if isinstance(file_template, dict) and \
+           isinstance(chalice_template, dict):
+            return self._merge_dict(file_template, chalice_template)
+        return file_template
+
+    def _merge_dict(self, file_template, chalice_template):
+        # type: (Dict[str, Any], Dict[str, Any]) -> Dict[str, Any]
+        merged = chalice_template.copy()
+        for key, value in file_template.items():
+            merged[key] = self._merge(value, chalice_template.get(key))
+        return merged

docs/source/topics/cfn.rst

@@ -36,6 +36,78 @@ your app, you won't be able to switch back to ``chalice deploy``.
 Running ``chalice deploy`` would create an entirely new set of AWS
 resources (API Gateway Rest API, AWS Lambda function, etc).
 
+Template Merge
+--------------
+
+It's a common use case to need to modify a Chalice generated template
+before deployment. Often to inject extra resources, values, or
+configurations that are not supported directly by Chalice. It will
+always be the case that something on AWS is not supported by Chalice
+directly that a consumer may want to interact with.
+
+The package command can now be invoked with the ``--merge-template`` argument::
+
+  $ chalice package --merge-template extras.json out
+
+This extras.json file should be a JSON formatted file which will be
+deep-merged on top of the sam.json that is generated by Chalice.
+
+For a simple example lets assume that we have the default new Chalice
+project and that extras.json has the following content::
+
+   {
+       "Resources": {
+           "APIHandler": {
+               "Properties": {
+                   "Environment": {
+                       "Variables": {
+                           "foo": "bar"
+                       }
+                   }
+               }
+           }
+       }
+   }
+
+
+The generated template located at out/sam.json will have this
+environment variable injected into it::
+
+    ...
+        "APIHandler": {
+          "Type": "AWS::Serverless::Function",
+          "Properties": {
+            "Runtime": "python3.6",
+            "Handler": "app.app",
+            "CodeUri": "./deployment.zip",
+            "Tags": {
+              "aws-chalice": "version=1.9.1:stage=dev:app=test"
+            },
+            "Timeout": 60,
+            "MemorySize": 128,
+            "Role": {
+              "Fn::GetAtt": [
+                "DefaultRole",
+                "Arn"
+              ]
+            },
+            "Environment": {
+              "Variables": {
+                "foo": "bar"
+              }
+            }
+          }
+        },
+    ...
+
+Obviously this simple example could have been achieved using the
+config file for the environment variables. But using the extras.json
+we have the ability to inject anything we want into the template file.
+A slightly more interesting example would be having an extras.json that
+injects a DynamoDB table into the resources section. And injects a REF to that
+table into the API handler lambda function's environment variables.
+
+
 Example
 -------
 
@@ -119,5 +191,3 @@ will be available as an output::
     {
         "hello": "world"
     }
-
-