FEATURE ASH-229: Add AWS Data Export COR suport

README.md

@@ -4,8 +4,9 @@ This repository provides CloudFormation templates helping to configure
 infrastructure in your account, including:
 
 - an IAM role allowing Vertice platform to access your resources,
-- an S3 bucket to store Cost and Usage Reports (CUR), and
-- a Cost and Usage Report definition.
+- an S3 bucket to store Cost and Usage Reports (CUR),
+- a Cost and Usage Report definition, and
+- an AWS Data Exports report definition (COR format).
 
 We also offer Terraform configuration of the same at
 [VerticeOne/terraform-aws-vertice-integration](https://github.com/VerticeOne/terraform-aws-vertice-integration),
@@ -15,8 +16,8 @@ granted to Vertice in your account.
 ## Using the template
 
 > [!IMPORTANT]  
-> Make sure you are in region **us-east-1**
-> See [Region Selection](#region-selection) for explanation
+> Make sure you are in the **us-east-1** region.
+> See [Region Selection](#region-selection) for explanation.
 
 The [Vertice governance CloudFormation template](templates/governance.yaml) is published
 to the following S3 address:
@@ -26,7 +27,7 @@ https://vertice-cco-cloudformation-templates.s3.eu-west-1.amazonaws.com/vertice-
 ```
 
 > [!TIP]
-> To deploy it, use this CloudFormation [quick-create link](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateURL=https://vertice-cco-cloudformation-templates.s3.eu-west-1.amazonaws.com/vertice-governance.yaml) and populate all of the fields.
+> To deploy the stack easily, use this CloudFormation [quick-create link](https://us-east-1.console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/quickcreate?templateURL=https://vertice-cco-cloudformation-templates.s3.eu-west-1.amazonaws.com/vertice-governance.yaml) and populate all of the fields.
 
 Alternatively, [create a new CloudFormation Stack](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-cli-creating-stack.html),
 providing the URL above as the `--template-url` parameter (or `Amazon S3 URL`
@@ -62,7 +63,7 @@ The Unique identifier provided to you for the Vertice platform to assume the IAM
 #### Optional parameters
 
 The creation of specific resources is further controlled by the
-**BillingReportCreate**, **BillingBucketCreate** and **VerticeIAMRoleCreate**
+**BillingReportCreate**, **BillingBucketCreate**, **DataExportCreate**, and **VerticeIAMRoleCreate**
 parameters.
 Please see the `Parameters` section of the template for further details.
 

templates/governance.yaml

@@ -1,6 +1,6 @@
 ---
 AWSTemplateFormatVersion: 2010-09-09
-Description: 'Set up Vertice governance infrastructure (IAM role, S3 bucket, CUR report).'
+Description: 'Set up Vertice governance infrastructure (IAM role, S3 bucket, CUR/COR report).'
 
 Parameters:
   AccountType:
@@ -40,6 +40,19 @@ Parameters:
     Default: 'Retain'
     Type: String
     AllowedValues: ['Delete', 'Retain', 'RetainExceptOnCreate']
+  DataExportCreate:
+    Description: 'Enable AWS Data Exports (COR format) and export results to defined S3 bucket.'
+    Default: 'false'
+    Type: String
+    AllowedValues: ['true', 'false']
+  DataExportName:
+    Description: 'Name for created AWS Data Export.'
+    Type: String
+    Default: 'vertice-cor-report'
+  DataExportS3Prefix:
+    Description: 'S3 bucket object name prefix for AWS Data Export output.'
+    Type: String
+    Default: 'cor'
   VerticeIAMRoleCreate:
     Description: "Create IAM role allowing Vertice to access your account's billing data."
     Default: 'true'
@@ -74,6 +87,9 @@ Conditions:
   CreateReport: !And
     - !Equals [!Ref BillingReportCreate, 'true']
     - !Condition BillingAccess
+  CreateDataExport: !And
+    - !Equals [!Ref DataExportCreate, 'true']
+    - !Condition BillingAccess
 
   CreateRole: !Equals [!Ref VerticeIAMRoleCreate, 'true']
   CreateBillingPolicy: !And
@@ -115,17 +131,20 @@ Resources:
             Action:
               - 's3:GetBucketAcl'
               - 's3:GetBucketPolicy'
+              - 's3:PutObject'
             Principal:
-              Service: ['billingreports.amazonaws.com']
+              Service:
+                - 'bcm-data-exports.amazonaws.com'
+                - 'billingreports.amazonaws.com'
             Resource:
               - !Sub 'arn:aws:s3:::${BillingBucketName}'
-          - Sid: AllowBillingBucketObjectAccess
-            Effect: 'Allow'
-            Action: 's3:PutObject'
-            Principal:
-              Service: ['billingreports.amazonaws.com']
-            Resource:
               - !Sub 'arn:aws:s3:::${BillingBucketName}/*'
+            Condition:
+              StringLike:
+                'aws:SourceArn':
+                  - !Sub 'arn:aws:bcm-data-exports:us-east-1:${AWS::AccountId}:export/*'
+                  - !Sub 'arn:aws:cur:us-east-1:${AWS::AccountId}:definition/*'
+                'aws:SourceAccount': !Sub ${AWS::AccountId}
 
   VerticeGovernanceBillingReport:
     Type: 'AWS::CUR::ReportDefinition'
@@ -150,6 +169,46 @@ Resources:
       S3Region: !Ref AWS::Region
       TimeUnit: 'HOURLY'
 
+  VerticeGovernanceDataExport:
+    Type: AWS::BCMDataExports::Export
+    Condition: CreateDataExport
+    Metadata:
+      # https://github.com/aws/serverless-application-model/issues/259#issuecomment-523953576
+      DependsOn: [!If [CreateBucket, !Ref VerticeGovernanceS3BucketPolicy, !Ref AWS::NoValue]]
+    Properties:
+      Export:
+        DataQuery:
+          QueryStatement: SELECT account_id, action_type, currency_code, current_resource_details,
+            current_resource_summary, current_resource_type, estimated_monthly_cost_after_discount,
+            estimated_monthly_cost_before_discount, estimated_monthly_savings_after_discount,
+            estimated_monthly_savings_before_discount, estimated_savings_percentage_after_discount,
+            estimated_savings_percentage_before_discount, implementation_effort, last_refresh_timestamp,
+            recommendation_id, recommendation_lookback_period_in_days, recommendation_source,
+            recommended_resource_details, recommended_resource_summary, recommended_resource_type,
+            region, resource_arn, restart_needed, rollback_possible, tags FROM COST_OPTIMIZATION_RECOMMENDATIONS
+          TableConfigurations:
+            COST_OPTIMIZATION_RECOMMENDATIONS:
+              FILTER: '{}'
+              INCLUDE_ALL_RECOMMENDATIONS: 'TRUE'
+        DestinationConfigurations:
+          S3Destination:
+            S3Bucket:
+              !If [
+                CreateBucket,
+                  !Ref VerticeGovernanceS3Bucket,
+                  !Ref BillingBucketName,
+                ]
+            S3OutputConfigurations:
+              Compression: 'PARQUET'
+              Format: 'PARQUET'
+              OutputType: 'CUSTOM'
+              Overwrite: 'OVERWRITE_REPORT'
+            S3Prefix: !Ref DataExportS3Prefix
+            S3Region: !Ref AWS::Region
+        Name: !Ref DataExportName
+        RefreshCadence:
+          Frequency: 'SYNCHRONOUS'
+
   VerticeGovernanceIAMRole:
     Type: 'AWS::IAM::Role'
     Condition: CreateRole
@@ -195,6 +254,10 @@ Resources:
           - Sid: 'VerticeBillingAccess'
             Effect: 'Allow'
             Action:
+              - 'bcm-data-exports:GetExport'
+              - 'bcm-data-exports:GetExecution'
+              - 'bcm-data-exports:ListExports'
+              - 'bcm-data-exports:ListExecutions'
               - 'budgets:Describe*'
               - 'budgets:View*'
               - 'ce:Describe*'
@@ -289,7 +352,7 @@ Resources:
 
 Outputs:
   VerticeGovernanceRoleArn:
-    Value: !GetAtt VerticeGovernanceIAMRole.Arn
+    Value: !If [CreateRole, !GetAtt VerticeGovernanceIAMRole.Arn, !Ref AWS::NoValue]
   FullS3Path:
     Value: !Join ["/", ["s3:/", !Ref BillingBucketName, !Ref BillingReportName, !Ref BillingReportS3Prefix ] ]
   VerticeIAMRoleAccountIDs: