From 36e4fb6138e86ef0e8d156810bc5b331e681ba98 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Tue, 2 May 2023 15:47:27 -0500 Subject: [PATCH 01/14] add s3_configuration to attributes --- internal/service/firehose/delivery_stream.go | 59 ++++++++++---------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index c17a4928126..74211d0b391 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -128,7 +128,7 @@ func s3ConfigurationSchema() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, MaxItems: 1, - Optional: true, + Required: true, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "bucket_arn": { @@ -1001,7 +1001,6 @@ func ResourceDeliveryStream() *schema.Resource { return strings.ToLower(value) }, ValidateFunc: validation.StringInSlice([]string{ - destinationTypeS3, destinationTypeExtendedS3, destinationTypeRedshift, destinationTypeElasticsearch, @@ -1011,8 +1010,6 @@ func ResourceDeliveryStream() *schema.Resource { }, false), }, - "s3_configuration": s3ConfigurationSchema(), - "extended_s3_configuration": { Type: schema.TypeList, Optional: true, @@ -1370,6 +1367,8 @@ func ResourceDeliveryStream() *schema.Resource { "s3_backup_configuration": s3ConfigurationSchema(), + "s3_configuration": s3ConfigurationSchema(), + "retry_duration": { Type: schema.TypeInt, Optional: true, @@ -1449,6 +1448,8 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: verify.ValidARN, }, + "s3_configuration": s3ConfigurationSchema(), + "s3_backup_mode": { Type: schema.TypeString, ForceNew: true, @@ -1560,6 +1561,8 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: verify.ValidARN, }, + "s3_configuration": s3ConfigurationSchema(), + "s3_backup_mode": { Type: schema.TypeString, ForceNew: true, @@ -1662,6 +1665,8 @@ func ResourceDeliveryStream() *schema.Resource { }, false), }, + "s3_configuration": s3ConfigurationSchema(), + "retry_duration": { Type: schema.TypeInt, Optional: true, @@ -1719,6 +1724,8 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: validation.StringInSlice(firehose.HttpEndpointS3BackupMode_Values(), false), }, + "s3_configuration": s3ConfigurationSchema(), + "retry_duration": { Type: schema.TypeInt, Optional: true, @@ -1779,8 +1786,8 @@ func createSourceConfig(source map[string]interface{}) *firehose.KinesisStreamSo return configuration } -func createS3Config(d *schema.ResourceData) *firehose.S3DestinationConfiguration { - s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) +func createS3Config(tfList []interface{}) *firehose.S3DestinationConfiguration { + s3 := tfList[0].(map[string]interface{}) configuration := &firehose.S3DestinationConfiguration{ BucketARN: aws.String(s3["bucket_arn"].(string)), @@ -2284,7 +2291,7 @@ func extractPrefixConfiguration(s3 map[string]interface{}) *string { return nil } -func createRedshiftConfig(d *schema.ResourceData, s3Config *firehose.S3DestinationConfiguration) (*firehose.RedshiftDestinationConfiguration, error) { +func createRedshiftConfig(d *schema.ResourceData) (*firehose.RedshiftDestinationConfiguration, error) { redshiftRaw, ok := d.GetOk("redshift_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("redshift_configuration", destinationTypeRedshift) @@ -2300,7 +2307,7 @@ func createRedshiftConfig(d *schema.ResourceData, s3Config *firehose.S3Destinati Username: aws.String(redshift["username"].(string)), RoleARN: aws.String(redshift["role_arn"].(string)), CopyCommand: extractCopyCommandConfiguration(redshift), - S3Configuration: s3Config, + S3Configuration: createS3Config(redshift["s3_configuration"].([]interface{})), } if _, ok := redshift["cloudwatch_logging_options"]; ok { @@ -2360,7 +2367,7 @@ func elasticsearchDestinationRequiredParamErr(param, destination string) error { return fmt.Errorf(`%q is required when "destination" is %q`, param, destination) } -func createElasticsearchConfig(d *schema.ResourceData, s3Config *firehose.S3DestinationConfiguration) (*firehose.ElasticsearchDestinationConfiguration, error) { +func createElasticsearchConfig(d *schema.ResourceData) (*firehose.ElasticsearchDestinationConfiguration, error) { esConfig, ok := d.GetOk("elasticsearch_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("elasticsearch_configuration", destinationTypeElasticsearch) @@ -2375,7 +2382,7 @@ func createElasticsearchConfig(d *schema.ResourceData, s3Config *firehose.S3Dest RetryOptions: extractElasticsearchRetryOptions(es), RoleARN: aws.String(es["role_arn"].(string)), TypeName: aws.String(es["type_name"].(string)), - S3Configuration: s3Config, + S3Configuration: createS3Config(es["s3_configuration"].([]interface{})), } if v, ok := es["domain_arn"]; ok && v.(string) != "" { @@ -2449,7 +2456,7 @@ func updateElasticsearchConfig(d *schema.ResourceData, s3Update *firehose.S3Dest return update, nil } -func createOpensearchConfig(d *schema.ResourceData, s3Config *firehose.S3DestinationConfiguration) (*firehose.AmazonopensearchserviceDestinationConfiguration, error) { +func createOpensearchConfig(d *schema.ResourceData) (*firehose.AmazonopensearchserviceDestinationConfiguration, error) { esConfig, ok := d.GetOk("opensearch_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("opensearch_configuration", destinationTypeOpensearch) @@ -2464,7 +2471,7 @@ func createOpensearchConfig(d *schema.ResourceData, s3Config *firehose.S3Destina RetryOptions: extractOpensearchRetryOptions(es), RoleARN: aws.String(es["role_arn"].(string)), TypeName: aws.String(es["type_name"].(string)), - S3Configuration: s3Config, + S3Configuration: createS3Config(es["s3_configuration"].([]interface{})), } if v, ok := es["domain_arn"]; ok && v.(string) != "" { @@ -2538,7 +2545,7 @@ func updateOpensearchConfig(d *schema.ResourceData, s3Update *firehose.S3Destina return update, nil } -func createSplunkConfig(d *schema.ResourceData, s3Config *firehose.S3DestinationConfiguration) (*firehose.SplunkDestinationConfiguration, error) { +func createSplunkConfig(d *schema.ResourceData) (*firehose.SplunkDestinationConfiguration, error) { splunkRaw, ok := d.GetOk("splunk_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("splunk_configuration", destinationTypeSplunk) @@ -2553,7 +2560,7 @@ func createSplunkConfig(d *schema.ResourceData, s3Config *firehose.S3Destination HECEndpoint: aws.String(splunk["hec_endpoint"].(string)), HECAcknowledgmentTimeoutInSeconds: aws.Int64(int64(splunk["hec_acknowledgment_timeout"].(int))), RetryOptions: extractSplunkRetryOptions(splunk), - S3Configuration: s3Config, + S3Configuration: createS3Config(splunk["s3_configuration"].([]interface{})), } if _, ok := splunk["processing_configuration"]; ok { @@ -2602,7 +2609,7 @@ func updateSplunkConfig(d *schema.ResourceData, s3Update *firehose.S3Destination return configuration, nil } -func createHTTPEndpointConfig(d *schema.ResourceData, s3Config *firehose.S3DestinationConfiguration) (*firehose.HttpEndpointDestinationConfiguration, error) { +func createHTTPEndpointConfig(d *schema.ResourceData) (*firehose.HttpEndpointDestinationConfiguration, error) { HttpEndpointRaw, ok := d.GetOk("http_endpoint_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("http_endpoint_configuration", destinationTypeHTTPEndpoint) @@ -2614,7 +2621,7 @@ func createHTTPEndpointConfig(d *schema.ResourceData, s3Config *firehose.S3Desti configuration := &firehose.HttpEndpointDestinationConfiguration{ RetryOptions: extractHTTPEndpointRetryOptions(HttpEndpoint), RoleARN: aws.String(HttpEndpoint["role_arn"].(string)), - S3Configuration: s3Config, + S3Configuration: createS3Config(HttpEndpoint["s3_configuration"].([]interface{})), } configuration.EndpointConfiguration = extractHTTPEndpointConfiguration(HttpEndpoint) @@ -2862,40 +2869,36 @@ func resourceDeliveryStreamCreate(ctx context.Context, d *schema.ResourceData, m extendedS3Config := createExtendedS3Config(d) input.ExtendedS3DestinationConfiguration = extendedS3Config } else { - s3Config := createS3Config(d) - - if d.Get("destination").(string) == destinationTypeS3 { - input.S3DestinationConfiguration = s3Config - } else if d.Get("destination").(string) == destinationTypeElasticsearch { - esConfig, err := createElasticsearchConfig(d, s3Config) + if d.Get("destination").(string) == destinationTypeElasticsearch { + esConfig, err := createElasticsearchConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "creating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } input.ElasticsearchDestinationConfiguration = esConfig } else if d.Get("destination").(string) == destinationTypeOpensearch { - osConfig, err := createOpensearchConfig(d, s3Config) + osConfig, err := createOpensearchConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "creating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } input.AmazonopensearchserviceDestinationConfiguration = osConfig } else if d.Get("destination").(string) == destinationTypeRedshift { - rc, err := createRedshiftConfig(d, s3Config) + rc, err := createRedshiftConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "creating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } input.RedshiftDestinationConfiguration = rc } else if d.Get("destination").(string) == destinationTypeSplunk { - rc, err := createSplunkConfig(d, s3Config) + spConfig, err := createSplunkConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "creating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } - input.SplunkDestinationConfiguration = rc + input.SplunkDestinationConfiguration = spConfig } else if d.Get("destination").(string) == destinationTypeHTTPEndpoint { - rc, err := createHTTPEndpointConfig(d, s3Config) + httpConfig, err := createHTTPEndpointConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "creating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } - input.HttpEndpointDestinationConfiguration = rc + input.HttpEndpointDestinationConfiguration = httpConfig } } From b330e375688c717903760503c2e8ecfa6d895826 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 10:20:39 -0500 Subject: [PATCH 02/14] s3_configuration in update func --- internal/service/firehose/delivery_stream.go | 52 +++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index 74211d0b391..c8b31eb42ca 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -1883,8 +1883,8 @@ func createExtendedS3Config(d *schema.ResourceData) *firehose.ExtendedS3Destinat return configuration } -func updateS3Config(d *schema.ResourceData) *firehose.S3DestinationUpdate { - s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) +func updateS3Config(tfList []interface{}) *firehose.S3DestinationUpdate { + s3 := tfList[0].(map[string]interface{}) configuration := &firehose.S3DestinationUpdate{ BucketARN: aws.String(s3["bucket_arn"].(string)), @@ -2324,7 +2324,7 @@ func createRedshiftConfig(d *schema.ResourceData) (*firehose.RedshiftDestination return configuration, nil } -func updateRedshiftConfig(d *schema.ResourceData, s3Update *firehose.S3DestinationUpdate) (*firehose.RedshiftDestinationUpdate, error) { +func updateRedshiftConfig(d *schema.ResourceData) (*firehose.RedshiftDestinationUpdate, error) { redshiftRaw, ok := d.GetOk("redshift_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("redshift_configuration", destinationTypeRedshift) @@ -2340,9 +2340,15 @@ func updateRedshiftConfig(d *schema.ResourceData, s3Update *firehose.S3Destinati Username: aws.String(redshift["username"].(string)), RoleARN: aws.String(redshift["role_arn"].(string)), CopyCommand: extractCopyCommandConfiguration(redshift), - S3Update: s3Update, } + s3Config := updateS3Config(redshift["s3_configuration"].([]interface{})) + // Redshift does not currently support ErrorOutputPrefix, + // which is set to the empty string within "updateS3Config", + // thus we must remove it here to avoid an InvalidArgumentException. + s3Config.ErrorOutputPrefix = nil + configuration.S3Update = s3Config + if _, ok := redshift["cloudwatch_logging_options"]; ok { configuration.CloudWatchLoggingOptions = extractCloudWatchLoggingConfiguration(redshift) } @@ -2415,7 +2421,7 @@ func createElasticsearchConfig(d *schema.ResourceData) (*firehose.ElasticsearchD return config, nil } -func updateElasticsearchConfig(d *schema.ResourceData, s3Update *firehose.S3DestinationUpdate) (*firehose.ElasticsearchDestinationUpdate, error) { +func updateElasticsearchConfig(d *schema.ResourceData) (*firehose.ElasticsearchDestinationUpdate, error) { esConfig, ok := d.GetOk("elasticsearch_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("elasticsearch_configuration", destinationTypeElasticsearch) @@ -2430,7 +2436,7 @@ func updateElasticsearchConfig(d *schema.ResourceData, s3Update *firehose.S3Dest RetryOptions: extractElasticsearchRetryOptions(es), RoleARN: aws.String(es["role_arn"].(string)), TypeName: aws.String(es["type_name"].(string)), - S3Update: s3Update, + S3Update: updateS3Config(es["s3_configuration"].([]interface{})), } if v, ok := es["domain_arn"]; ok && v.(string) != "" { @@ -2504,7 +2510,7 @@ func createOpensearchConfig(d *schema.ResourceData) (*firehose.Amazonopensearchs return config, nil } -func updateOpensearchConfig(d *schema.ResourceData, s3Update *firehose.S3DestinationUpdate) (*firehose.AmazonopensearchserviceDestinationUpdate, error) { +func updateOpensearchConfig(d *schema.ResourceData) (*firehose.AmazonopensearchserviceDestinationUpdate, error) { esConfig, ok := d.GetOk("opensearch_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("elasticsearch_configuration", destinationTypeOpensearch) @@ -2519,7 +2525,7 @@ func updateOpensearchConfig(d *schema.ResourceData, s3Update *firehose.S3Destina RetryOptions: extractOpensearchRetryOptions(es), RoleARN: aws.String(es["role_arn"].(string)), TypeName: aws.String(es["type_name"].(string)), - S3Update: s3Update, + S3Update: updateS3Config(es["s3_configuration"].([]interface{})), } if v, ok := es["domain_arn"]; ok && v.(string) != "" { @@ -2577,7 +2583,7 @@ func createSplunkConfig(d *schema.ResourceData) (*firehose.SplunkDestinationConf return configuration, nil } -func updateSplunkConfig(d *schema.ResourceData, s3Update *firehose.S3DestinationUpdate) (*firehose.SplunkDestinationUpdate, error) { +func updateSplunkConfig(d *schema.ResourceData) (*firehose.SplunkDestinationUpdate, error) { splunkRaw, ok := d.GetOk("splunk_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("splunk_configuration", destinationTypeSplunk) @@ -2592,7 +2598,7 @@ func updateSplunkConfig(d *schema.ResourceData, s3Update *firehose.S3Destination HECEndpoint: aws.String(splunk["hec_endpoint"].(string)), HECAcknowledgmentTimeoutInSeconds: aws.Int64(int64(splunk["hec_acknowledgment_timeout"].(int))), RetryOptions: extractSplunkRetryOptions(splunk), - S3Update: s3Update, + S3Update: updateS3Config(splunk["s3_configuration"].([]interface{})), } if _, ok := splunk["processing_configuration"]; ok { @@ -2654,7 +2660,7 @@ func createHTTPEndpointConfig(d *schema.ResourceData) (*firehose.HttpEndpointDes return configuration, nil } -func updateHTTPEndpointConfig(d *schema.ResourceData, s3Update *firehose.S3DestinationUpdate) (*firehose.HttpEndpointDestinationUpdate, error) { +func updateHTTPEndpointConfig(d *schema.ResourceData) (*firehose.HttpEndpointDestinationUpdate, error) { HttpEndpointRaw, ok := d.GetOk("http_endpoint_configuration") if !ok { return nil, elasticsearchDestinationRequiredParamErr("http_endpoint_configuration", destinationTypeHTTPEndpoint) @@ -2666,7 +2672,7 @@ func updateHTTPEndpointConfig(d *schema.ResourceData, s3Update *firehose.S3Desti configuration := &firehose.HttpEndpointDestinationUpdate{ RetryOptions: extractHTTPEndpointRetryOptions(HttpEndpoint), RoleARN: aws.String(HttpEndpoint["role_arn"].(string)), - S3Update: s3Update, + S3Update: updateS3Config(HttpEndpoint["s3_configuration"].([]interface{})), } configuration.EndpointConfiguration = extractHTTPEndpointConfiguration(HttpEndpoint) @@ -3020,42 +3026,32 @@ func resourceDeliveryStreamUpdate(ctx context.Context, d *schema.ResourceData, m extendedS3Config := updateExtendedS3Config(d) updateInput.ExtendedS3DestinationUpdate = extendedS3Config } else { - s3Config := updateS3Config(d) - - if d.Get("destination").(string) == destinationTypeS3 { - updateInput.S3DestinationUpdate = s3Config - } else if d.Get("destination").(string) == destinationTypeElasticsearch { - esUpdate, err := updateElasticsearchConfig(d, s3Config) + if d.Get("destination").(string) == destinationTypeElasticsearch { + esUpdate, err := updateElasticsearchConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "updating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } updateInput.ElasticsearchDestinationUpdate = esUpdate } else if d.Get("destination").(string) == destinationTypeOpensearch { - esUpdate, err := updateOpensearchConfig(d, s3Config) + esUpdate, err := updateOpensearchConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "updating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } updateInput.AmazonopensearchserviceDestinationUpdate = esUpdate } else if d.Get("destination").(string) == destinationTypeRedshift { - // Redshift does not currently support ErrorOutputPrefix, - // which is set to the empty string within "updateS3Config", - // thus we must remove it here to avoid an InvalidArgumentException. - if s3Config != nil { - s3Config.ErrorOutputPrefix = nil - } - rc, err := updateRedshiftConfig(d, s3Config) + rc, err := updateRedshiftConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "updating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } updateInput.RedshiftDestinationUpdate = rc } else if d.Get("destination").(string) == destinationTypeSplunk { - rc, err := updateSplunkConfig(d, s3Config) + rc, err := updateSplunkConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "updating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } updateInput.SplunkDestinationUpdate = rc } else if d.Get("destination").(string) == destinationTypeHTTPEndpoint { - rc, err := updateHTTPEndpointConfig(d, s3Config) + rc, err := updateHTTPEndpointConfig(d) if err != nil { return sdkdiag.AppendErrorf(diags, "updating Kinesis Firehose Delivery Stream (%s): %s", sn, err) } From db7086edfe1268bbce05d5253db4281f4386c7c9 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 10:31:20 -0500 Subject: [PATCH 03/14] s3_configuration in read --- internal/service/firehose/delivery_stream.go | 25 ++++---------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index c8b31eb42ca..4963ada2525 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -261,6 +261,7 @@ func flattenElasticsearchConfiguration(description *firehose.ElasticsearchDestin "type_name": aws.StringValue(description.TypeName), "index_name": aws.StringValue(description.IndexName), "s3_backup_mode": aws.StringValue(description.S3BackupMode), + "s3_configuration": flattenS3Configuration(description.S3DestinationDescription), "index_rotation_period": aws.StringValue(description.IndexRotationPeriod), "vpc_config": flattenVPCConfiguration(description.VpcConfigurationDescription), "processing_configuration": flattenProcessingConfiguration(description.ProcessingConfiguration, aws.StringValue(description.RoleARN)), @@ -297,6 +298,7 @@ func flattenOpensearchConfiguration(description *firehose.Amazonopensearchservic "type_name": aws.StringValue(description.TypeName), "index_name": aws.StringValue(description.IndexName), "s3_backup_mode": aws.StringValue(description.S3BackupMode), + "s3_configuration": flattenS3Configuration(description.S3DestinationDescription), "index_rotation_period": aws.StringValue(description.IndexRotationPeriod), "vpc_config": flattenVPCConfiguration(description.VpcConfigurationDescription), "processing_configuration": flattenProcessingConfiguration(description.ProcessingConfiguration, aws.StringValue(description.RoleARN)), @@ -381,6 +383,7 @@ func flattenRedshiftConfiguration(description *firehose.RedshiftDestinationDescr "role_arn": aws.StringValue(description.RoleARN), "s3_backup_configuration": flattenS3Configuration(description.S3BackupDescription), "s3_backup_mode": aws.StringValue(description.S3BackupMode), + "s3_configuration": flattenS3Configuration(description.S3DestinationDescription), "username": aws.StringValue(description.Username), } @@ -409,6 +412,7 @@ func flattenSplunkConfiguration(description *firehose.SplunkDestinationDescripti "hec_token": aws.StringValue(description.HECToken), "processing_configuration": flattenProcessingConfiguration(description.ProcessingConfiguration, ""), "s3_backup_mode": aws.StringValue(description.S3BackupMode), + "s3_configuration": flattenS3Configuration(description.S3DestinationDescription), } if description.RetryOptions != nil { @@ -808,47 +812,27 @@ func flattenDeliveryStream(d *schema.ResourceData, s *firehose.DeliveryStreamDes if err := d.Set("redshift_configuration", flattenRedshiftConfiguration(destination.RedshiftDestinationDescription, configuredPassword)); err != nil { return fmt.Errorf("setting redshift_configuration: %s", err) } - if err := d.Set("s3_configuration", flattenS3Configuration(destination.RedshiftDestinationDescription.S3DestinationDescription)); err != nil { - return fmt.Errorf("setting s3_configuration: %s", err) - } } else if destination.ElasticsearchDestinationDescription != nil { d.Set("destination", destinationTypeElasticsearch) if err := d.Set("elasticsearch_configuration", flattenElasticsearchConfiguration(destination.ElasticsearchDestinationDescription)); err != nil { return fmt.Errorf("setting elasticsearch_configuration: %s", err) } - if err := d.Set("s3_configuration", flattenS3Configuration(destination.ElasticsearchDestinationDescription.S3DestinationDescription)); err != nil { - return fmt.Errorf("setting s3_configuration: %s", err) - } } else if destination.AmazonopensearchserviceDestinationDescription != nil { d.Set("destination", destinationTypeOpensearch) if err := d.Set("opensearch_configuration", flattenOpensearchConfiguration(destination.AmazonopensearchserviceDestinationDescription)); err != nil { return fmt.Errorf("setting opensearch_configuration: %s", err) } - if err := d.Set("s3_configuration", flattenS3Configuration(destination.AmazonopensearchserviceDestinationDescription.S3DestinationDescription)); err != nil { - return fmt.Errorf("setting s3_configuration: %s", err) - } } else if destination.SplunkDestinationDescription != nil { d.Set("destination", destinationTypeSplunk) if err := d.Set("splunk_configuration", flattenSplunkConfiguration(destination.SplunkDestinationDescription)); err != nil { return fmt.Errorf("setting splunk_configuration: %s", err) } - if err := d.Set("s3_configuration", flattenS3Configuration(destination.SplunkDestinationDescription.S3DestinationDescription)); err != nil { - return fmt.Errorf("setting s3_configuration: %s", err) - } } else if destination.HttpEndpointDestinationDescription != nil { d.Set("destination", destinationTypeHTTPEndpoint) configuredAccessKey := d.Get("http_endpoint_configuration.0.access_key").(string) if err := d.Set("http_endpoint_configuration", flattenHTTPEndpointConfiguration(destination.HttpEndpointDestinationDescription, configuredAccessKey)); err != nil { return fmt.Errorf("setting http_endpoint_configuration: %s", err) } - if err := d.Set("s3_configuration", flattenS3Configuration(destination.HttpEndpointDestinationDescription.S3DestinationDescription)); err != nil { - return fmt.Errorf("setting s3_configuration: %s", err) - } - } else if d.Get("destination").(string) == destinationTypeS3 { - d.Set("destination", destinationTypeS3) - if err := d.Set("s3_configuration", flattenS3Configuration(destination.S3DestinationDescription)); err != nil { - return fmt.Errorf("setting s3_configuration: %s", err) - } } else { d.Set("destination", destinationTypeExtendedS3) if err := d.Set("extended_s3_configuration", flattenExtendedS3Configuration(destination.ExtendedS3DestinationDescription)); err != nil { @@ -871,6 +855,7 @@ func flattenHTTPEndpointConfiguration(description *firehose.HttpEndpointDestinat "name": aws.StringValue(description.EndpointConfiguration.Name), "role_arn": aws.StringValue(description.RoleARN), "s3_backup_mode": aws.StringValue(description.S3BackupMode), + "s3_configuration": flattenS3Configuration(description.S3DestinationDescription), "request_configuration": flattenRequestConfiguration(description.RequestConfiguration), "cloudwatch_logging_options": flattenCloudWatchLoggingOptions(description.CloudWatchLoggingOptions), "processing_configuration": flattenProcessingConfiguration(description.ProcessingConfiguration, aws.StringValue(description.RoleARN)), From 7c5baff42ee89cd9a11772f71cf85e2a3815d2a6 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 10:40:46 -0500 Subject: [PATCH 04/14] arrange alphabetically --- internal/service/firehose/delivery_stream.go | 61 ++++++++++---------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index 4963ada2525..594e9572eb9 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -1319,12 +1319,24 @@ func ResourceDeliveryStream() *schema.Resource { MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ + "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), + "cluster_jdbcurl": { Type: schema.TypeString, Required: true, }, - "username": { + "copy_options": { + Type: schema.TypeString, + Optional: true, + }, + + "data_table_columns": { + Type: schema.TypeString, + Optional: true, + }, + + "data_table_name": { Type: schema.TypeString, Required: true, }, @@ -1337,6 +1349,13 @@ func ResourceDeliveryStream() *schema.Resource { "processing_configuration": processingConfigurationSchema(), + "retry_duration": { + Type: schema.TypeInt, + Optional: true, + Default: 3600, + ValidateFunc: validation.IntBetween(0, 7200), + }, + "role_arn": { Type: schema.TypeString, Required: true, @@ -1354,29 +1373,10 @@ func ResourceDeliveryStream() *schema.Resource { "s3_configuration": s3ConfigurationSchema(), - "retry_duration": { - Type: schema.TypeInt, - Optional: true, - Default: 3600, - ValidateFunc: validation.IntBetween(0, 7200), - }, - - "copy_options": { - Type: schema.TypeString, - Optional: true, - }, - - "data_table_columns": { - Type: schema.TypeString, - Optional: true, - }, - - "data_table_name": { + "username": { Type: schema.TypeString, Required: true, }, - - "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), }, }, }, @@ -1401,6 +1401,14 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: validation.IntBetween(1, 100), }, + "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), + + "cluster_endpoint": { + Type: schema.TypeString, + Optional: true, + ConflictsWith: []string{"elasticsearch_configuration.0.domain_arn"}, + }, + "domain_arn": { Type: schema.TypeString, Optional: true, @@ -1420,6 +1428,8 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: validation.StringInSlice(firehose.ElasticsearchIndexRotationPeriod_Values(), false), }, + "processing_configuration": processingConfigurationSchema(), + "retry_duration": { Type: schema.TypeInt, Optional: true, @@ -1481,15 +1491,6 @@ func ResourceDeliveryStream() *schema.Resource { }, }, }, - - "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), - - "processing_configuration": processingConfigurationSchema(), - "cluster_endpoint": { - Type: schema.TypeString, - Optional: true, - ConflictsWith: []string{"elasticsearch_configuration.0.domain_arn"}, - }, }, }, }, From 034057ec0dc2ce68a792067c06569d0edcc7f0d2 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 10:43:04 -0500 Subject: [PATCH 05/14] arrange alphabetically --- internal/service/firehose/delivery_stream.go | 28 ++++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index 594e9572eb9..e6c91e457f0 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -1641,6 +1641,13 @@ func ResourceDeliveryStream() *schema.Resource { Required: true, }, + "retry_duration": { + Type: schema.TypeInt, + Optional: true, + Default: 3600, + ValidateFunc: validation.IntBetween(0, 7200), + }, + "s3_backup_mode": { Type: schema.TypeString, Optional: true, @@ -1653,13 +1660,6 @@ func ResourceDeliveryStream() *schema.Resource { "s3_configuration": s3ConfigurationSchema(), - "retry_duration": { - Type: schema.TypeInt, - Optional: true, - Default: 3600, - ValidateFunc: validation.IntBetween(0, 7200), - }, - "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), "processing_configuration": processingConfigurationSchema(), @@ -1697,6 +1697,13 @@ func ResourceDeliveryStream() *schema.Resource { Sensitive: true, }, + "retry_duration": { + Type: schema.TypeInt, + Optional: true, + Default: 300, + ValidateFunc: validation.IntBetween(0, 7200), + }, + "role_arn": { Type: schema.TypeString, Optional: true, @@ -1712,13 +1719,6 @@ func ResourceDeliveryStream() *schema.Resource { "s3_configuration": s3ConfigurationSchema(), - "retry_duration": { - Type: schema.TypeInt, - Optional: true, - Default: 300, - ValidateFunc: validation.IntBetween(0, 7200), - }, - "buffering_interval": { Type: schema.TypeInt, Optional: true, From acce44d8d6454688f037b72efff418e5066908ab Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 11:05:42 -0500 Subject: [PATCH 06/14] cleanup: remove test for s3 destination --- internal/service/firehose/delivery_stream.go | 7 +- .../service/firehose/delivery_stream_test.go | 544 ++---------------- 2 files changed, 66 insertions(+), 485 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index e6c91e457f0..1735cb4f9a7 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -996,10 +996,9 @@ func ResourceDeliveryStream() *schema.Resource { }, "extended_s3_configuration": { - Type: schema.TypeList, - Optional: true, - ConflictsWith: []string{"s3_configuration"}, - MaxItems: 1, + Type: schema.TypeList, + Optional: true, + MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "bucket_arn": { diff --git a/internal/service/firehose/delivery_stream_test.go b/internal/service/firehose/delivery_stream_test.go index c8b8ba16f62..601dd58ca35 100644 --- a/internal/service/firehose/delivery_stream_test.go +++ b/internal/service/firehose/delivery_stream_test.go @@ -125,249 +125,6 @@ func TestAccFirehoseDeliveryStream_tags(t *testing.T) { }) } -func TestAccFirehoseDeliveryStream_s3basic(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3basic(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - ), - }, - }, - }) -} - -func TestAccFirehoseDeliveryStream_s3basicWithPrefixes(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3basicPrefixes(rName, "logs/", ""), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.prefix", "logs/"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", ""), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicPrefixes(rName, "logs/", "errors/"), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.prefix", "logs/"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "errors/"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicPrefixes(rName, "logs/{timestamp:yyyy-MM-dd}/", "errors/{timestamp:yyyy-MM-dd}/"), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.prefix", "logs/{timestamp:yyyy-MM-dd}/"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "errors/{timestamp:yyyy-MM-dd}/"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicPrefixes(rName, "", ""), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.prefix", ""), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", ""), - ), - }, - }, - }) -} - -func TestAccFirehoseDeliveryStream_s3basicWithSSE(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3basicSSE(rName, true), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "true"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicSSE(rName, false), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basic(rName), - PlanOnly: true, - }, - { - Config: testAccDeliveryStreamConfig_s3basicSSE(rName, true), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "true"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", "AWS_OWNED_CMK"), - ), - }, - }, - }) -} - -func TestAccFirehoseDeliveryStream_s3basicWithSSEAndKeyARN(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3BasicSSEAndKeyARN(rName, true), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "true"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", firehose.KeyTypeCustomerManagedCmk), - resource.TestCheckResourceAttrPair(resourceName, "server_side_encryption.0.key_arn", "aws_kms_key.test", "arn"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicSSE(rName, false), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3BasicSSEAndKeyARN(rName, true), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "true"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", firehose.KeyTypeCustomerManagedCmk), - resource.TestCheckResourceAttrPair(resourceName, "server_side_encryption.0.key_arn", "aws_kms_key.test", "arn"), - ), - }, - }, - }) -} - -func TestAccFirehoseDeliveryStream_s3basicWithSSEAndKeyType(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3basicSSEAndKeyType(rName, true, firehose.KeyTypeAwsOwnedCmk), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "true"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", firehose.KeyTypeAwsOwnedCmk), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicSSE(rName, false), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "false"), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3basicSSEAndKeyType(rName, true, firehose.KeyTypeAwsOwnedCmk), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.#", "1"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.enabled", "true"), - resource.TestCheckResourceAttr(resourceName, "server_side_encryption.0.key_type", firehose.KeyTypeAwsOwnedCmk), - ), - }, - }, - }) -} - -func TestAccFirehoseDeliveryStream_s3KinesisStreamSource(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3Source(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - ), - }, - }, - }) -} - func TestAccFirehoseDeliveryStream_s3WithCloudWatchLogging(t *testing.T) { ctx := acctest.Context(t) var stream firehose.DeliveryStreamDescription @@ -391,43 +148,6 @@ func TestAccFirehoseDeliveryStream_s3WithCloudWatchLogging(t *testing.T) { }) } -func TestAccFirehoseDeliveryStream_s3Updates(t *testing.T) { - ctx := acctest.Context(t) - var stream firehose.DeliveryStreamDescription - rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) - resourceName := "aws_kinesis_firehose_delivery_stream.test" - - updatedS3DestinationConfig := &firehose.S3DestinationDescription{ - BufferingHints: &firehose.BufferingHints{ - IntervalInSeconds: aws.Int64(400), - SizeInMBs: aws.Int64(10), - }, - } - - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(ctx, t) }, - ErrorCheck: acctest.ErrorCheck(t, firehose.EndpointsID), - ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, - CheckDestroy: testAccCheckDeliveryStreamDestroy(ctx), - Steps: []resource.TestStep{ - { - Config: testAccDeliveryStreamConfig_s3basic(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - ), - }, - { - Config: testAccDeliveryStreamConfig_s3Updates(rName), - Check: resource.ComposeTestCheckFunc( - testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - testAccCheckDeliveryStreamAttributes(&stream, updatedS3DestinationConfig, nil, nil, nil, nil, nil, nil), - ), - }, - }, - }) -} - func TestAccFirehoseDeliveryStream_extendedS3basic(t *testing.T) { ctx := acctest.Context(t) var stream firehose.DeliveryStreamDescription @@ -2471,9 +2191,9 @@ resource "aws_cloudwatch_log_stream" "test" { resource "aws_kinesis_firehose_delivery_stream" "test" { depends_on = [aws_iam_role_policy.firehose] name = %[1]q - destination = "s3" + destination = "extended_s3" - s3_configuration { + extended_s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn @@ -2487,111 +2207,14 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { `, rName) } -func testAccDeliveryStreamConfig_s3basic(rName string) string { - return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = [aws_iam_role_policy.firehose] - name = %[1]q - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } -} -`, rName)) -} - -func testAccDeliveryStreamConfig_s3basicPrefixes(rName, prefix, errorOutputPrefix string) string { - return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = ["aws_iam_role_policy.firehose"] - name = %[1]q - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - prefix = %[2]q - error_output_prefix = %[3]q - } -} -`, rName, prefix, errorOutputPrefix)) -} - -func testAccDeliveryStreamConfig_s3basicSSE(rName string, sseEnabled bool) string { - return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = [aws_iam_role_policy.firehose] - name = %[1]q - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - - server_side_encryption { - enabled = %[2]t - } -} -`, rName, sseEnabled)) -} - -func testAccDeliveryStreamConfig_s3BasicSSEAndKeyARN(rName string, sseEnabled bool) string { - return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` -resource "aws_kms_key" "test" { - deletion_window_in_days = 7 - description = %[1]q -} - -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = [aws_iam_role_policy.firehose] - name = %[1]q - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - - server_side_encryption { - enabled = %[2]t - key_arn = aws_kms_key.test.arn - key_type = "CUSTOMER_MANAGED_CMK" - } -} -`, rName, sseEnabled)) -} - -func testAccDeliveryStreamConfig_s3basicSSEAndKeyType(rName string, sseEnabled bool, keyType string) string { - return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = [aws_iam_role_policy.firehose] - name = %[1]q - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - - server_side_encryption { - enabled = %[2]t - key_type = %[3]q - } -} -`, rName, sseEnabled, keyType)) -} - func testAccDeliveryStreamConfig_tags1(rName, tagKey1, tagValue1 string) string { return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` resource "aws_kinesis_firehose_delivery_stream" "test" { depends_on = [aws_iam_role_policy.firehose] name = %[1]q - destination = "s3" + destination = "extended_s3" - s3_configuration { + extended_s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn } @@ -2608,9 +2231,9 @@ func testAccDeliveryStreamConfig_tags2(rName, tagKey1, tagValue1, tagKey2, tagVa resource "aws_kinesis_firehose_delivery_stream" "test" { depends_on = [aws_iam_role_policy.firehose] name = %[1]q - destination = "s3" + destination = "extended_s3" - s3_configuration { + extended_s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn } @@ -2623,48 +2246,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { `, rName, tagKey1, tagValue1, tagKey2, tagValue2)) } -func testAccDeliveryStreamConfig_s3Source(rName string) string { - return acctest.ConfigCompose( - testAccDeliveryStreamConfig_base(rName), - testAccDeliveryStreamConfig_baseKinesisStreamSource(rName), - fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = [aws_iam_role_policy.firehose, aws_iam_role_policy.kinesis_source] - name = %[1]q - - kinesis_source_configuration { - kinesis_stream_arn = aws_kinesis_stream.source.arn - role_arn = aws_iam_role.kinesis_source.arn - } - - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } -} -`, rName)) -} - -func testAccDeliveryStreamConfig_s3Updates(rName string) string { - return acctest.ConfigCompose(testAccDeliveryStreamConfig_base(rName), fmt.Sprintf(` -resource "aws_kinesis_firehose_delivery_stream" "test" { - depends_on = [aws_iam_role_policy.firehose] - name = %[1]q - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } -} -`, rName)) -} - func testAccDeliveryStreamConfig_extendedS3basic(rName string) string { return acctest.ConfigCompose( testAccDeliveryStreamConfig_baseLambda(rName), @@ -3351,17 +2932,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "redshift" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - redshift_configuration { role_arn = aws_iam_role.firehose.arn cluster_jdbcurl = "jdbc:redshift://${aws_redshift_cluster.test.endpoint}/${aws_redshift_cluster.test.database_name}" username = "testuser" password = "T3stPass" data_table_name = "test-table" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3376,14 +2957,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "redshift" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - redshift_configuration { role_arn = aws_iam_role.firehose.arn cluster_jdbcurl = "jdbc:redshift://${aws_redshift_cluster.test.endpoint}/${aws_redshift_cluster.test.database_name}" @@ -3396,6 +2969,15 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { bucket_arn = aws_s3_bucket.bucket.arn } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + + data_table_name = "test-table" copy_options = "GZIP" data_table_columns = "test-col" @@ -3432,14 +3014,14 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "splunk" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - splunk_configuration { hec_endpoint = "https://input-test.com:443" hec_token = "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3455,14 +3037,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "splunk" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - splunk_configuration { hec_endpoint = "https://input-test.com:443" hec_token = "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A" @@ -3470,6 +3044,14 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { hec_endpoint_type = "Event" s3_backup_mode = "FailedEventsOnly" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + processing_configuration { enabled = true @@ -3509,15 +3091,15 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "splunk" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - error_output_prefix = %[2]q - } - splunk_configuration { hec_endpoint = "https://input-test.com:443" hec_token = "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + error_output_prefix = %[2]q + } } } `, rName, errorOutputPrefix)) @@ -3530,15 +3112,15 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "http_endpoint" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - http_endpoint_configuration { url = "https://input-test.com:443" name = "HTTP_test" role_arn = aws_iam_role.firehose.arn + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3551,16 +3133,16 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "http_endpoint" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - error_output_prefix = %[2]q - } - http_endpoint_configuration { url = "https://input-test.com:443" name = "HTTP_test" role_arn = aws_iam_role.firehose.arn + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + error_output_prefix = %[2]q + } } } `, rName, errorOutputPrefix)) @@ -3573,16 +3155,16 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "http_endpoint" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - http_endpoint_configuration { url = "https://input-test.com:443" name = "HTTP_test" retry_duration = %[2]d role_arn = aws_iam_role.firehose.arn + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName, retryDuration)) @@ -3598,14 +3180,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "http_endpoint" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - http_endpoint_configuration { url = "https://input-test.com:443" name = "HTTP_test" @@ -3613,6 +3187,14 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn s3_backup_mode = "FailedDataOnly" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + processing_configuration { enabled = true From 82b1f91bbe08b11530ddad0e7eb94a0b75414ffe Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 12:45:14 -0500 Subject: [PATCH 07/14] test: check correct attribute locations --- internal/service/firehose/delivery_stream.go | 125 ++++++------ .../service/firehose/delivery_stream_test.go | 190 +++++++++--------- 2 files changed, 156 insertions(+), 159 deletions(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index 1735cb4f9a7..a6a14dd48b7 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -124,65 +124,78 @@ func requestConfigurationSchema() *schema.Schema { } } +func s3BackupConfigurationSchema() *schema.Schema { + return &schema.Schema{ + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + Elem: s3ConfigurationElem(), + } +} + func s3ConfigurationSchema() *schema.Schema { return &schema.Schema{ Type: schema.TypeList, MaxItems: 1, Required: true, - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "bucket_arn": { - Type: schema.TypeString, - Required: true, - ValidateFunc: verify.ValidARN, - }, + Elem: s3ConfigurationElem(), + } +} - "buffering_size": { - Type: schema.TypeInt, - Optional: true, - Default: 5, - ValidateFunc: validation.IntAtLeast(1), - }, +func s3ConfigurationElem() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "bucket_arn": { + Type: schema.TypeString, + Required: true, + ValidateFunc: verify.ValidARN, + }, - "buffering_interval": { - Type: schema.TypeInt, - Optional: true, - Default: 300, - ValidateFunc: validation.IntAtLeast(60), - }, + "buffering_size": { + Type: schema.TypeInt, + Optional: true, + Default: 5, + ValidateFunc: validation.IntAtLeast(1), + }, - "compression_format": { - Type: schema.TypeString, - Optional: true, - Default: firehose.CompressionFormatUncompressed, - ValidateFunc: validation.StringInSlice(firehose.CompressionFormat_Values(), false), - }, + "buffering_interval": { + Type: schema.TypeInt, + Optional: true, + Default: 300, + ValidateFunc: validation.IntAtLeast(60), + }, - "error_output_prefix": { - Type: schema.TypeString, - Optional: true, - ValidateFunc: validation.StringLenBetween(0, 1024), - }, + "compression_format": { + Type: schema.TypeString, + Optional: true, + Default: firehose.CompressionFormatUncompressed, + ValidateFunc: validation.StringInSlice(firehose.CompressionFormat_Values(), false), + }, - "kms_key_arn": { - Type: schema.TypeString, - Optional: true, - ValidateFunc: verify.ValidARN, - }, + "error_output_prefix": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringLenBetween(0, 1024), + }, - "role_arn": { - Type: schema.TypeString, - Required: true, - ValidateFunc: verify.ValidARN, - }, + "kms_key_arn": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: verify.ValidARN, + }, - "prefix": { - Type: schema.TypeString, - Optional: true, - }, + "role_arn": { + Type: schema.TypeString, + Required: true, + ValidateFunc: verify.ValidARN, + }, - "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), + "prefix": { + Type: schema.TypeString, + Optional: true, }, + + "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), }, } } @@ -1301,7 +1314,7 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: validation.StringInSlice(firehose.S3BackupMode_Values(), false), }, - "s3_backup_configuration": s3ConfigurationSchema(), + "s3_backup_configuration": s3BackupConfigurationSchema(), "cloudwatch_logging_options": cloudWatchLoggingOptionsSchema(), @@ -1368,7 +1381,7 @@ func ResourceDeliveryStream() *schema.Resource { ValidateFunc: validation.StringInSlice(firehose.S3BackupMode_Values(), false), }, - "s3_backup_configuration": s3ConfigurationSchema(), + "s3_backup_configuration": s3BackupConfigurationSchema(), "s3_configuration": s3ConfigurationSchema(), @@ -2960,28 +2973,12 @@ func resourceDeliveryStreamCreate(ctx context.Context, d *schema.ResourceData, m } func validateSchema(d *schema.ResourceData) error { - _, s3Exists := d.GetOk("s3_configuration") _, extendedS3Exists := d.GetOk("extended_s3_configuration") if d.Get("destination").(string) == destinationTypeExtendedS3 { if !extendedS3Exists { return fmt.Errorf( - "When destination is 'extended_s3', extended_s3_configuration is required", - ) - } else if s3Exists { - return fmt.Errorf( - "When destination is 'extended_s3', s3_configuration must not be set", - ) - } - } else { - if !s3Exists { - return fmt.Errorf( - "When destination is %s, s3_configuration is required", - d.Get("destination").(string), - ) - } else if extendedS3Exists { - return fmt.Errorf( - "extended_s3_configuration can only be used when destination is 'extended_s3'", + "when destination is 'extended_s3', extended_s3_configuration is required", ) } } diff --git a/internal/service/firehose/delivery_stream_test.go b/internal/service/firehose/delivery_stream_test.go index 601dd58ca35..4b19019312f 100644 --- a/internal/service/firehose/delivery_stream_test.go +++ b/internal/service/firehose/delivery_stream_test.go @@ -956,8 +956,8 @@ func TestAccFirehoseDeliveryStream_Splunk_ErrorOutputPrefix(t *testing.T) { Config: testAccDeliveryStreamConfig_splunkErrorOutputPrefix(rName, "prefix1"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix1"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.0.s3_configuration.0.error_output_prefix", "prefix1"), ), }, { @@ -969,8 +969,8 @@ func TestAccFirehoseDeliveryStream_Splunk_ErrorOutputPrefix(t *testing.T) { Config: testAccDeliveryStreamConfig_splunkErrorOutputPrefix(rName, "prefix2"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix2"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.0.s3_configuration.0.error_output_prefix", "prefix2"), ), }, { @@ -982,8 +982,8 @@ func TestAccFirehoseDeliveryStream_Splunk_ErrorOutputPrefix(t *testing.T) { Config: testAccDeliveryStreamConfig_splunkErrorOutputPrefix(rName, ""), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", ""), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "splunk_configuration.0.s3_configuration.0.error_output_prefix", ""), ), }, }, @@ -1063,8 +1063,8 @@ func TestAccFirehoseDeliveryStream_HTTPEndpoint_ErrorOutputPrefix(t *testing.T) Config: testAccDeliveryStreamConfig_httpEndpointErrorOutputPrefix(rName, "prefix1"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix1"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.0.s3_configuration.0.error_output_prefix", "prefix1"), ), }, { @@ -1076,8 +1076,8 @@ func TestAccFirehoseDeliveryStream_HTTPEndpoint_ErrorOutputPrefix(t *testing.T) Config: testAccDeliveryStreamConfig_httpEndpointErrorOutputPrefix(rName, "prefix2"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix2"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.0.s3_configuration.0.error_output_prefix", "prefix2"), ), }, { @@ -1089,8 +1089,8 @@ func TestAccFirehoseDeliveryStream_HTTPEndpoint_ErrorOutputPrefix(t *testing.T) Config: testAccDeliveryStreamConfig_httpEndpointErrorOutputPrefix(rName, ""), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", ""), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "http_endpoint_configuration.0.s3_configuration.0.error_output_prefix", ""), ), }, }, @@ -1321,8 +1321,8 @@ func TestAccFirehoseDeliveryStream_Elasticsearch_ErrorOutputPrefix(t *testing.T) Config: testAccDeliveryStreamConfig_elasticsearchErrorOutputPrefix(rName, "prefix1"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix1"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.s3_configuration.0.error_output_prefix", "prefix1"), ), }, { @@ -1334,8 +1334,8 @@ func TestAccFirehoseDeliveryStream_Elasticsearch_ErrorOutputPrefix(t *testing.T) Config: testAccDeliveryStreamConfig_elasticsearchErrorOutputPrefix(rName, "prefix2"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix2"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.s3_configuration.0.error_output_prefix", "prefix2"), ), }, { @@ -1347,8 +1347,8 @@ func TestAccFirehoseDeliveryStream_Elasticsearch_ErrorOutputPrefix(t *testing.T) Config: testAccDeliveryStreamConfig_elasticsearchErrorOutputPrefix(rName, ""), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", ""), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.s3_configuration.0.error_output_prefix", ""), ), }, }, @@ -1558,8 +1558,8 @@ func TestAccFirehoseDeliveryStream_Opensearch_ErrorOutputPrefix(t *testing.T) { Config: testAccDeliveryStreamConfig_opensearchErrorOutputPrefix(rName, "prefix1"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix1"), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.0.s3_configuration.0.error_output_prefix", "prefix1"), ), }, { @@ -1571,8 +1571,8 @@ func TestAccFirehoseDeliveryStream_Opensearch_ErrorOutputPrefix(t *testing.T) { Config: testAccDeliveryStreamConfig_opensearchErrorOutputPrefix(rName, "prefix2"), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", "prefix2"), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.0.s3_configuration.0.error_output_prefix", "prefix2"), ), }, { @@ -1584,8 +1584,8 @@ func TestAccFirehoseDeliveryStream_Opensearch_ErrorOutputPrefix(t *testing.T) { Config: testAccDeliveryStreamConfig_opensearchErrorOutputPrefix(rName, ""), Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.#", "1"), - resource.TestCheckResourceAttr(resourceName, "s3_configuration.0.error_output_prefix", ""), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.0.s3_configuration.#", "1"), + resource.TestCheckResourceAttr(resourceName, "opensearch_configuration.0.s3_configuration.0.error_output_prefix", ""), ), }, }, @@ -3346,16 +3346,16 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" type_name = "test" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3369,17 +3369,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - error_output_prefix = %[2]q - } - elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" type_name = "test" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + error_output_prefix = %[2]q + } } } `, rName, errorOutputPrefix)) @@ -3393,17 +3393,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" type_name = "test" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + vpc_config { subnet_ids = aws_subnet.test[*].id security_group_ids = aws_security_group.test[*].id @@ -3425,11 +3425,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn @@ -3437,6 +3432,11 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { type_name = "test" buffering_interval = 500 + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + processing_configuration { enabled = false @@ -3473,11 +3473,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn @@ -3485,6 +3480,11 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { type_name = "test" buffering_interval = 500 + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + vpc_config { subnet_ids = aws_subnet.test[*].id security_group_ids = aws_security_group.test[*].id @@ -3521,16 +3521,16 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - elasticsearch_configuration { cluster_endpoint = "https://${aws_elasticsearch_domain.test_cluster.endpoint}" role_arn = aws_iam_role.firehose.arn index_name = "test" type_name = "test" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } }`, rName)) } @@ -3546,11 +3546,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - elasticsearch_configuration { cluster_endpoint = "https://${aws_elasticsearch_domain.test_cluster.endpoint}" role_arn = aws_iam_role.firehose.arn @@ -3558,6 +3553,11 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { type_name = "test" buffering_interval = 500 + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + processing_configuration { enabled = false @@ -3706,15 +3706,15 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - opensearch_configuration { domain_arn = aws_opensearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3728,15 +3728,15 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - opensearch_configuration { cluster_endpoint = "https://${aws_opensearch_domain.test_cluster.endpoint}" role_arn = aws_iam_role.firehose.arn index_name = "test" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } }`, rName)) } @@ -3752,17 +3752,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - opensearch_configuration { cluster_endpoint = "https://${aws_opensearch_domain.test_cluster.endpoint}" role_arn = aws_iam_role.firehose.arn index_name = "test" buffering_interval = 500 + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + processing_configuration { enabled = false @@ -3795,16 +3795,16 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" + opensearch_configuration { + domain_arn = aws_opensearch_domain.test_cluster.arn + role_arn = aws_iam_role.firehose.arn + index_name = "test" + s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn error_output_prefix = %[2]q } - - opensearch_configuration { - domain_arn = aws_opensearch_domain.test_cluster.arn - role_arn = aws_iam_role.firehose.arn - index_name = "test" } } `, rName, errorOutputPrefix)) @@ -3818,16 +3818,16 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - opensearch_configuration { domain_arn = aws_opensearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + vpc_config { subnet_ids = aws_subnet.test[*].id security_group_ids = aws_security_group.test[*].id @@ -3849,17 +3849,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - opensearch_configuration { domain_arn = aws_opensearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" buffering_interval = 500 + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + processing_configuration { enabled = false @@ -3896,17 +3896,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = %[1]q destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } - opensearch_configuration { domain_arn = aws_opensearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" buffering_interval = 500 + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + vpc_config { subnet_ids = aws_subnet.test[*].id security_group_ids = aws_security_group.test[*].id From 5b1742f32c04b2f1905359c81ff86b3fcdb283da Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 16:19:37 -0500 Subject: [PATCH 08/14] linter: fix acctest fmt --- .../service/firehose/delivery_stream_test.go | 116 +++++++++--------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/internal/service/firehose/delivery_stream_test.go b/internal/service/firehose/delivery_stream_test.go index 4b19019312f..0def277192d 100644 --- a/internal/service/firehose/delivery_stream_test.go +++ b/internal/service/firehose/delivery_stream_test.go @@ -3352,10 +3352,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" type_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3375,11 +3375,11 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" type_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - error_output_prefix = %[2]q - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + error_output_prefix = %[2]q + } } } `, rName, errorOutputPrefix)) @@ -3399,10 +3399,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" type_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } vpc_config { subnet_ids = aws_subnet.test[*].id @@ -3432,10 +3432,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { type_name = "test" buffering_interval = 500 - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } processing_configuration { enabled = false @@ -3480,10 +3480,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { type_name = "test" buffering_interval = 500 - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } vpc_config { subnet_ids = aws_subnet.test[*].id @@ -3527,10 +3527,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" type_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } }`, rName)) } @@ -3553,10 +3553,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { type_name = "test" buffering_interval = 500 - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } processing_configuration { enabled = false @@ -3711,10 +3711,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn index_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } } `, rName)) @@ -3733,10 +3733,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn index_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } } }`, rName)) } @@ -3758,10 +3758,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" buffering_interval = 500 - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } processing_configuration { enabled = false @@ -3800,11 +3800,11 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn index_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - error_output_prefix = %[2]q - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + error_output_prefix = %[2]q + } } } `, rName, errorOutputPrefix)) @@ -3823,10 +3823,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn index_name = "test" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } vpc_config { subnet_ids = aws_subnet.test[*].id @@ -3855,10 +3855,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" buffering_interval = 500 - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } processing_configuration { enabled = false @@ -3902,10 +3902,10 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { index_name = "test" buffering_interval = 500 - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } vpc_config { subnet_ids = aws_subnet.test[*].id From a4ab9f39ec687c7b98f42794c3fd20f289a62ed0 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 16:33:35 -0500 Subject: [PATCH 09/14] fmt --- internal/service/firehose/delivery_stream_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/service/firehose/delivery_stream_test.go b/internal/service/firehose/delivery_stream_test.go index 0def277192d..ddb2dcd4c6c 100644 --- a/internal/service/firehose/delivery_stream_test.go +++ b/internal/service/firehose/delivery_stream_test.go @@ -2972,8 +2972,8 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 + buffering_size = 10 + buffering_interval = 400 compression_format = "GZIP" } @@ -3047,8 +3047,8 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 + buffering_size = 10 + buffering_interval = 400 compression_format = "GZIP" } @@ -3190,8 +3190,8 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { s3_configuration { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 + buffering_size = 10 + buffering_interval = 400 compression_format = "GZIP" } From 755dc694ee93e0e03f4d7285cb919ef84557e464 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 16:46:31 -0500 Subject: [PATCH 10/14] add CHANGELOG entry --- .changelog/31138.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .changelog/31138.txt diff --git a/.changelog/31138.txt b/.changelog/31138.txt new file mode 100644 index 00000000000..66426549653 --- /dev/null +++ b/.changelog/31138.txt @@ -0,0 +1,11 @@ +```release-note:breaking-change +resource/aws_kinesis_firehose_delivery_stream: Remove `s3` as an option for `destination`. Use `extended_s3` instead +``` + +```release-note:breaking-change +resource/aws_kinesis_firehose_delivery_stream: Remove `s3_configuration` attribute from the root of the resource. `s3_configuration` is now a part of the following blocks: `elasticsearch_configuration`, `opensearch_configuration`, `redshift_configuration`, `splunk_configuration`, and `http_endpoint_configuration` +``` + +```release-note:enhancement +resource/aws_kinesis_firehose_delivery_stream: Add `s3_configuration` to `elasticsearch_configuration`, `opensearch_configuration`, `redshift_configuration`, `splunk_configuration`, and `http_endpoint_configuration` +``` \ No newline at end of file From 7728255bc7950003a942aa7dbc223eba87126257 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 17:04:34 -0500 Subject: [PATCH 11/14] docs: update documentation --- ...sis_firehose_delivery_stream.html.markdown | 179 +++++++----------- 1 file changed, 72 insertions(+), 107 deletions(-) diff --git a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown index 32735d67f7d..88622ef89c6 100644 --- a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown @@ -152,47 +152,6 @@ resource "aws_kinesis_firehose_delivery_stream" "extended_s3_stream" { } ``` -### S3 Destination (deprecated) - -```terraform -resource "aws_s3_bucket" "bucket" { - bucket = "tf-test-bucket" -} - -resource "aws_s3_bucket_acl" "bucket_acl" { - bucket = aws_s3_bucket.bucket.id - acl = "private" -} - -data "aws_iam_policy_document" "assume_role" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["firehose.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - -resource "aws_iam_role" "firehose_role" { - name = "firehose_test_role" - assume_role_policy = data.aws_iam_policy_document.assume_role.json -} - -resource "aws_kinesis_firehose_delivery_stream" "test_stream" { - name = "terraform-kinesis-firehose-test-stream" - destination = "s3" - - s3_configuration { - role_arn = aws_iam_role.firehose_role.arn - bucket_arn = aws_s3_bucket.bucket.arn - } -} -``` - ### Redshift Destination ```terraform @@ -209,14 +168,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { name = "terraform-kinesis-firehose-test-stream" destination = "redshift" - s3_configuration { - role_arn = aws_iam_role.firehose_role.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - redshift_configuration { role_arn = aws_iam_role.firehose_role.arn cluster_jdbcurl = "jdbc:redshift://${aws_redshift_cluster.test_cluster.endpoint}/${aws_redshift_cluster.test_cluster.database_name}" @@ -227,6 +178,14 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { data_table_columns = "test-col" s3_backup_mode = "Enabled" + s3_configuration { + role_arn = aws_iam_role.firehose_role.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + s3_backup_configuration { role_arn = aws_iam_role.firehose_role.arn bucket_arn = aws_s3_bucket.bucket.arn @@ -249,20 +208,20 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { name = "terraform-kinesis-firehose-test-stream" destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose_role.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose_role.arn index_name = "test" type_name = "test" + s3_configuration { + role_arn = aws_iam_role.firehose_role.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + processing_configuration { enabled = "true" @@ -342,16 +301,18 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = "terraform-kinesis-firehose-es" destination = "elasticsearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + elasticsearch_configuration { domain_arn = aws_elasticsearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" type_name = "test" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + vpc_config { subnet_ids = [aws_subnet.first.id, aws_subnet.second.id] security_group_ids = [aws_security_group.first.id] @@ -372,19 +333,19 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { name = "terraform-kinesis-firehose-test-stream" destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose_role.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - opensearch_configuration { domain_arn = aws_opensearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose_role.arn index_name = "test" + s3_configuration { + role_arn = aws_iam_role.firehose_role.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + processing_configuration { enabled = "true" @@ -467,15 +428,17 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { name = "terraform-kinesis-firehose-os" destination = "opensearch" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - } + opensearch_configuration { domain_arn = aws_opensearch_domain.test_cluster.arn role_arn = aws_iam_role.firehose.arn index_name = "test" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + } + vpc_config { subnet_ids = [aws_subnet.first.id, aws_subnet.second.id] security_group_ids = [aws_security_group.first.id] @@ -492,20 +455,20 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { name = "terraform-kinesis-firehose-test-stream" destination = "splunk" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - splunk_configuration { hec_endpoint = "https://http-inputs-mydomain.splunkcloud.com:443" hec_token = "51D4DA16-C61B-4F5F-8EC7-ED4301342A4A" hec_acknowledgment_timeout = 600 hec_endpoint_type = "Event" s3_backup_mode = "FailedEventsOnly" + + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } } } ``` @@ -517,14 +480,6 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { name = "terraform-kinesis-firehose-test-stream" destination = "http_endpoint" - s3_configuration { - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.bucket.arn - buffering_size = 10 - buffering_interval = 400 - compression_format = "GZIP" - } - http_endpoint_configuration { url = "https://aws-api.newrelic.com/firehose/v1" name = "New Relic" @@ -534,6 +489,14 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { role_arn = aws_iam_role.firehose.arn s3_backup_mode = "FailedDataOnly" + s3_configuration { + role_arn = aws_iam_role.firehose.arn + bucket_arn = aws_s3_bucket.bucket.arn + buffering_size = 10 + buffering_interval = 400 + compression_format = "GZIP" + } + request_configuration { content_encoding = "GZIP" @@ -561,7 +524,6 @@ The following arguments are supported: * `server_side_encryption` - (Optional) Encrypt at rest options. Server-side encryption should not be enabled when a kinesis stream is configured as the source of the firehose delivery stream. * `destination` – (Required) This is the destination to where the data is delivered. The only options are `s3` (Deprecated, use `extended_s3` instead), `extended_s3`, `redshift`, `elasticsearch`, `splunk`, `http_endpoint` and `opensearch`. -* `s3_configuration` - (Optional) Required for non-S3 destinations. For S3 destination, use `extended_s3_configuration` instead. Configuration options for the s3 destination (or the intermediate bucket if the destination is redshift). More details are given below. * `extended_s3_configuration` - (Optional, only Required when `destination` is `extended_s3`) Enhanced configuration options for the s3 destination. More details are given below. * `redshift_configuration` - (Optional) Configuration options if redshift is the destination. @@ -583,23 +545,7 @@ The `server_side_encryption` object supports the following: * `key_type`- (Optional) Type of encryption key. Default is `AWS_OWNED_CMK`. Valid values are `AWS_OWNED_CMK` and `CUSTOMER_MANAGED_CMK` * `key_arn` - (Optional) Amazon Resource Name (ARN) of the encryption key. Required when `key_type` is `CUSTOMER_MANAGED_CMK`. -The `s3_configuration` object supports the following: - -~> **NOTE:** This configuration block is deprecated for the `s3` destination. - -* `role_arn` - (Required) The ARN of the AWS credentials. -* `bucket_arn` - (Required) The ARN of the S3 bucket -* `prefix` - (Optional) The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket -* `buffering_size` - (Optional) Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. - We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher. -* `buffering_interval` - (Optional) Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300. -* `compression_format` - (Optional) The compression format. If no value is specified, the default is `UNCOMPRESSED`. Other supported values are `GZIP`, `ZIP`, `Snappy`, & `HADOOP_SNAPPY`. -* `error_output_prefix` - (Optional) Prefix added to failed records before writing them to S3. Not currently supported for `redshift` destination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see [Custom Prefixes for Amazon S3 Objects](https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). -* `kms_key_arn` - (Optional) Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will -be used. -* `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below - -The `extended_s3_configuration` object supports the same fields from `s3_configuration` as well as the following: +The `extended_s3_configuration` object supports the same fields from [s3_configuration](#s3configuration) as well as the following: * `data_format_conversion_configuration` - (Optional) Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below. * `processing_configuration` - (Optional) The data processing configuration. More details are given below. @@ -614,6 +560,7 @@ The `redshift_configuration` object supports the following: * `password` - (Required) The password for the username above. * `retry_duration` - (Optional) The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value. * `role_arn` - (Required) The arn of the role the stream assumes. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. * `s3_backup_mode` - (Optional) The Amazon S3 backup mode. Valid values are `Disabled` and `Enabled`. Default value is `Disabled`. * `s3_backup_configuration` - (Optional) The configuration for backup in Amazon S3. Required if `s3_backup_mode` is `Enabled`. Supports the same fields as `s3_configuration` object. * `data_table_name` - (Required) The name of the table in the redshift cluster that the s3 bucket will copy to. @@ -632,6 +579,7 @@ The `elasticsearch_configuration` object supports the following: * `index_rotation_period` - (Optional) The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are `NoRotation`, `OneHour`, `OneDay`, `OneWeek`, and `OneMonth`. The default value is `OneDay`. * `retry_duration` - (Optional) After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. * `role_arn` - (Required) The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for `DescribeElasticsearchDomain`, `DescribeElasticsearchDomains`, and `DescribeElasticsearchDomainConfig`. The pattern needs to be `arn:.*`. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedDocumentsOnly` and `AllDocuments`. Default value is `FailedDocumentsOnly`. * `type_name` - (Optional) The Elasticsearch type name with maximum length of 100 characters. * `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below @@ -648,6 +596,7 @@ The `opensearch_configuration` object supports the following: * `index_rotation_period` - (Optional) The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are `NoRotation`, `OneHour`, `OneDay`, `OneWeek`, and `OneMonth`. The default value is `OneDay`. * `retry_duration` - (Optional) After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. * `role_arn` - (Required) The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for `DescribeDomain`, `DescribeDomains`, and `DescribeDomainConfig`. The pattern needs to be `arn:.*`. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedDocumentsOnly` and `AllDocuments`. Default value is `FailedDocumentsOnly`. * `type_name` - (Optional) The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty. * `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below @@ -660,6 +609,7 @@ The `splunk_configuration` objects supports the following: * `hec_endpoint` - (Required) The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data. * `hec_endpoint_type` - (Optional) The HEC endpoint type. Valid values are `Raw` or `Event`. The default value is `Raw`. * `hec_token` - (Required) The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedEventsOnly` and `AllEvents`. Default value is `FailedEventsOnly`. * `retry_duration` - (Optional) After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. * `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below. @@ -671,6 +621,7 @@ The `http_endpoint_configuration` objects supports the following: * `name` - (Optional) The HTTP endpoint name. * `access_key` - (Optional) The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination. * `role_arn` - (Required) Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be `arn:.*`. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedDataOnly` and `AllData`. Default value is `FailedDataOnly`. * `buffering_size` - (Optional) Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. * `buffering_interval` - (Optional) Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes). @@ -760,6 +711,20 @@ resource "aws_kinesis_firehose_delivery_stream" "example" { * `schema_configuration` - (Required) Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below. * `enabled` - (Optional) Defaults to `true`. Set it to `false` if you want to disable format conversion while preserving the configuration details. +#### s3_configuration + +* `role_arn` - (Required) The ARN of the AWS credentials. +* `bucket_arn` - (Required) The ARN of the S3 bucket +* `prefix` - (Optional) The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket +* `buffering_size` - (Optional) Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. + We recommend setting SizeInMBs to a value greater than the amount of data you typically ingest into the delivery stream in 10 seconds. For example, if you typically ingest data at 1 MB/sec set SizeInMBs to be 10 MB or higher. +* `buffering_interval` - (Optional) Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300. +* `compression_format` - (Optional) The compression format. If no value is specified, the default is `UNCOMPRESSED`. Other supported values are `GZIP`, `ZIP`, `Snappy`, & `HADOOP_SNAPPY`. +* `error_output_prefix` - (Optional) Prefix added to failed records before writing them to S3. Not currently supported for `redshift` destination. This prefix appears immediately following the bucket name. For information about how to specify this prefix, see [Custom Prefixes for Amazon S3 Objects](https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). +* `kms_key_arn` - (Optional) Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will + be used. +* `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below + #### input_format_configuration * `deserializer` - (Required) Nested argument that specifies which deserializer to use. You can choose either the Apache Hive JSON SerDe or the OpenX JSON SerDe. More details below. From 4a0dd95addbcdd08248440f56a547cac6755295d Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 17:16:43 -0500 Subject: [PATCH 12/14] linter: website lint fix --- .../r/kinesis_firehose_delivery_stream.html.markdown | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown index 88622ef89c6..6070538fd28 100644 --- a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown @@ -185,7 +185,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { buffering_interval = 400 compression_format = "GZIP" } - + s3_backup_configuration { role_arn = aws_iam_role.firehose_role.arn bucket_arn = aws_s3_bucket.bucket.arn @@ -221,7 +221,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { buffering_interval = 400 compression_format = "GZIP" } - + processing_configuration { enabled = "true" @@ -312,7 +312,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn } - + vpc_config { subnet_ids = [aws_subnet.first.id, aws_subnet.second.id] security_group_ids = [aws_security_group.first.id] @@ -345,7 +345,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { buffering_interval = 400 compression_format = "GZIP" } - + processing_configuration { enabled = "true" @@ -438,7 +438,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test" { role_arn = aws_iam_role.firehose.arn bucket_arn = aws_s3_bucket.bucket.arn } - + vpc_config { subnet_ids = [aws_subnet.first.id, aws_subnet.second.id] security_group_ids = [aws_security_group.first.id] @@ -496,7 +496,7 @@ resource "aws_kinesis_firehose_delivery_stream" "test_stream" { buffering_interval = 400 compression_format = "GZIP" } - + request_configuration { content_encoding = "GZIP" From f122b0d73c1fe723f2f39c9cba3145f111625753 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 17:29:27 -0500 Subject: [PATCH 13/14] fix link in docs --- internal/service/firehose/delivery_stream_test.go | 4 ++-- .../kinesis_firehose_delivery_stream.html.markdown | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/service/firehose/delivery_stream_test.go b/internal/service/firehose/delivery_stream_test.go index ddb2dcd4c6c..c3c364e42c1 100644 --- a/internal/service/firehose/delivery_stream_test.go +++ b/internal/service/firehose/delivery_stream_test.go @@ -1279,7 +1279,7 @@ func TestAccFirehoseDeliveryStream_elasticSearchWithVPCUpdates(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, nil, nil, nil, nil), - resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.vpc_id", "aws_vpc.elasticsearch_in_vpc", "id"), + resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.vpc_id", "aws_vpc.test", "id"), resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.subnet_ids.#", "2"), resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.security_group_ids.#", "2"), resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.role_arn", "aws_iam_role.firehose", "arn"), @@ -1295,7 +1295,7 @@ func TestAccFirehoseDeliveryStream_elasticSearchWithVPCUpdates(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckDeliveryStreamExists(ctx, resourceName, &stream), testAccCheckDeliveryStreamAttributes(&stream, nil, nil, nil, updatedElasticsearchConfig, nil, nil, nil), - resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.vpc_id", "aws_vpc.elasticsearch_in_vpc", "id"), + resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.vpc_id", "aws_vpc.test", "id"), resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.subnet_ids.#", "2"), resource.TestCheckResourceAttr(resourceName, "elasticsearch_configuration.0.vpc_config.0.security_group_ids.#", "2"), resource.TestCheckResourceAttrPair(resourceName, "elasticsearch_configuration.0.vpc_config.0.role_arn", "aws_iam_role.firehose", "arn"), diff --git a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown index 6070538fd28..0bbfe569316 100644 --- a/website/docs/r/kinesis_firehose_delivery_stream.html.markdown +++ b/website/docs/r/kinesis_firehose_delivery_stream.html.markdown @@ -545,7 +545,7 @@ The `server_side_encryption` object supports the following: * `key_type`- (Optional) Type of encryption key. Default is `AWS_OWNED_CMK`. Valid values are `AWS_OWNED_CMK` and `CUSTOMER_MANAGED_CMK` * `key_arn` - (Optional) Amazon Resource Name (ARN) of the encryption key. Required when `key_type` is `CUSTOMER_MANAGED_CMK`. -The `extended_s3_configuration` object supports the same fields from [s3_configuration](#s3configuration) as well as the following: +The `extended_s3_configuration` object supports the same fields from [s3_configuration](#s3-configuration) as well as the following: * `data_format_conversion_configuration` - (Optional) Nested argument for the serializer, deserializer, and schema for converting data from the JSON format to the Parquet or ORC format before writing it to Amazon S3. More details given below. * `processing_configuration` - (Optional) The data processing configuration. More details are given below. @@ -560,7 +560,7 @@ The `redshift_configuration` object supports the following: * `password` - (Required) The password for the username above. * `retry_duration` - (Optional) The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt. The default value is 3600 seconds (60 minutes). Firehose does not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery attempt takes longer than the current value. * `role_arn` - (Required) The arn of the role the stream assumes. -* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3-configuration) for more details. * `s3_backup_mode` - (Optional) The Amazon S3 backup mode. Valid values are `Disabled` and `Enabled`. Default value is `Disabled`. * `s3_backup_configuration` - (Optional) The configuration for backup in Amazon S3. Required if `s3_backup_mode` is `Enabled`. Supports the same fields as `s3_configuration` object. * `data_table_name` - (Required) The name of the table in the redshift cluster that the s3 bucket will copy to. @@ -579,7 +579,7 @@ The `elasticsearch_configuration` object supports the following: * `index_rotation_period` - (Optional) The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are `NoRotation`, `OneHour`, `OneDay`, `OneWeek`, and `OneMonth`. The default value is `OneDay`. * `retry_duration` - (Optional) After an initial failure to deliver to Amazon Elasticsearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. * `role_arn` - (Required) The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for `DescribeElasticsearchDomain`, `DescribeElasticsearchDomains`, and `DescribeElasticsearchDomainConfig`. The pattern needs to be `arn:.*`. -* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3-configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedDocumentsOnly` and `AllDocuments`. Default value is `FailedDocumentsOnly`. * `type_name` - (Optional) The Elasticsearch type name with maximum length of 100 characters. * `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below @@ -596,7 +596,7 @@ The `opensearch_configuration` object supports the following: * `index_rotation_period` - (Optional) The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data. Valid values are `NoRotation`, `OneHour`, `OneDay`, `OneWeek`, and `OneMonth`. The default value is `OneDay`. * `retry_duration` - (Optional) After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. * `role_arn` - (Required) The ARN of the IAM role to be assumed by Firehose for calling the Amazon ES Configuration API and for indexing documents. The IAM role must have permission for `DescribeDomain`, `DescribeDomains`, and `DescribeDomainConfig`. The pattern needs to be `arn:.*`. -* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3-configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedDocumentsOnly` and `AllDocuments`. Default value is `FailedDocumentsOnly`. * `type_name` - (Optional) The Elasticsearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty. * `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below @@ -609,7 +609,7 @@ The `splunk_configuration` objects supports the following: * `hec_endpoint` - (Required) The HTTP Event Collector (HEC) endpoint to which Kinesis Firehose sends your data. * `hec_endpoint_type` - (Optional) The HEC endpoint type. Valid values are `Raw` or `Event`. The default value is `Raw`. * `hec_token` - (Required) The GUID that you obtain from your Splunk cluster when you create a new HEC endpoint. -* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3-configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedEventsOnly` and `AllEvents`. Default value is `FailedEventsOnly`. * `retry_duration` - (Optional) After an initial failure to deliver to Splunk, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. * `cloudwatch_logging_options` - (Optional) The CloudWatch Logging Options for the delivery stream. More details are given below. @@ -621,7 +621,7 @@ The `http_endpoint_configuration` objects supports the following: * `name` - (Optional) The HTTP endpoint name. * `access_key` - (Optional) The access key required for Kinesis Firehose to authenticate with the HTTP endpoint selected as the destination. * `role_arn` - (Required) Kinesis Data Firehose uses this IAM role for all the permissions that the delivery stream needs. The pattern needs to be `arn:.*`. -* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3configuration) for more details. +* `s3_configuration` - (Required) The S3 Configuration. See [s3_configuration](#s3-configuration) for more details. * `s3_backup_mode` - (Optional) Defines how documents should be delivered to Amazon S3. Valid values are `FailedDataOnly` and `AllData`. Default value is `FailedDataOnly`. * `buffering_size` - (Optional) Buffer incoming data to the specified size, in MBs, before delivering it to the destination. The default value is 5. * `buffering_interval` - (Optional) Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination. The default value is 300 (5 minutes). @@ -711,7 +711,7 @@ resource "aws_kinesis_firehose_delivery_stream" "example" { * `schema_configuration` - (Required) Nested argument that specifies the AWS Glue Data Catalog table that contains the column information. More details below. * `enabled` - (Optional) Defaults to `true`. Set it to `false` if you want to disable format conversion while preserving the configuration details. -#### s3_configuration +#### S3 Configuration * `role_arn` - (Required) The ARN of the AWS credentials. * `bucket_arn` - (Required) The ARN of the S3 bucket From b19ee4ccf7b3093aaa918f130411c0083f427be1 Mon Sep 17 00:00:00 2001 From: Adrian Johnson Date: Wed, 3 May 2023 21:42:36 -0400 Subject: [PATCH 14/14] remove s3 destination constant --- internal/service/firehose/delivery_stream.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/service/firehose/delivery_stream.go b/internal/service/firehose/delivery_stream.go index a6a14dd48b7..c9853197fd8 100644 --- a/internal/service/firehose/delivery_stream.go +++ b/internal/service/firehose/delivery_stream.go @@ -26,7 +26,6 @@ import ( ) const ( - destinationTypeS3 = "s3" destinationTypeExtendedS3 = "extended_s3" destinationTypeElasticsearch = "elasticsearch" destinationTypeOpensearch = "opensearch"