Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[exporter/datadogexporter] Migrating datadog exporter to use aws sdk v2 #36797

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

[exporter/datadogexporter] Migrating datadog exporter to use aws sdk v2 #36797

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1a27699
Migrating datadog exporter to use aws sdk v2
LZiHaN Dec 12, 2024
cbf993d
Merge branch 'main' into migrate-to-sdk-v2
LZiHaN Dec 12, 2024
da0c4cf
fix conflict
LZiHaN Dec 12, 2024
42aa861
Merge branch 'main' into migrate-to-sdk-v2
LZiHaN Dec 12, 2024
27f8cd2
fix lints
LZiHaN Dec 13, 2024
ce5d382
Merge branch 'migrate-to-sdk-v2' of github.com:LZiHaN/opentelemetry-c…
LZiHaN Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion exporter/datadogexporter/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ require (
github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.21.0
github.com/DataDog/sketches-go v1.4.6 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/cenkalti/backoff/v4 v4.3.0
github.com/google/go-cmp v0.6.0
github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.115.0
Expand Down Expand Up @@ -91,6 +91,10 @@ require (
)

require (
github.com/aws/aws-sdk-go-v2 v1.32.6
github.com/aws/aws-sdk-go-v2/config v1.28.6
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21
github.com/aws/aws-sdk-go-v2/service/ec2 v1.196.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/datadog v0.115.0
go.opentelemetry.io/collector/component/componenttest v0.115.1-0.20241206185113-3f3e208e71b8
go.opentelemetry.io/collector/consumer/consumererror v0.115.1-0.20241206185113-3f3e208e71b8
Expand Down Expand Up @@ -158,6 +162,16 @@ require (
github.com/antchfx/xmlquery v1.4.2 // indirect
github.com/antchfx/xpath v1.3.2 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.7.1 // indirect
Expand Down
28 changes: 28 additions & 0 deletions exporter/datadogexporter/go.sum
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

70 changes: 41 additions & 29 deletions exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib
import (
"context"
"fmt"
"io"
"strings"
"sync"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"

"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"

"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/hostmetadata/provider"
Expand Down Expand Up @@ -42,31 +46,42 @@ func isDefaultHostname(hostname string) bool {

// GetHostInfo gets the hostname info from EC2 metadata
func GetHostInfo(ctx context.Context, logger *zap.Logger) (hostInfo *HostInfo) {
sess, err := session.NewSession()
hostInfo = &HostInfo{}

cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
logger.Warn("Failed to build AWS session", zap.Error(err))
logger.Warn("Failed to build AWS config", zap.Error(err))
return
}

meta := ec2metadata.New(sess)
client := imds.NewFromConfig(cfg)

if !meta.AvailableWithContext(ctx) {
logger.Debug("EC2 Metadata not available")
// Check if metadata service is available by trying to retrieve instance ID
_, err = client.GetMetadata(ctx, &imds.GetMetadataInput{
Path: "instance-id",
})
if err != nil {
logger.Debug("EC2 Metadata service is not available", zap.Error(err))
return
}

if idDoc, err := meta.GetInstanceIdentityDocumentWithContext(ctx); err == nil {
if idDoc, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{}); err == nil {
hostInfo.InstanceID = idDoc.InstanceID
} else {
logger.Warn("Failed to get EC2 instance id document", zap.Error(err))
}

if ec2Hostname, err := meta.GetMetadataWithContext(ctx, "hostname"); err == nil {
hostInfo.EC2Hostname = ec2Hostname
metadataOutput, err := client.GetMetadata(ctx, &imds.GetMetadataInput{Path: "hostname"})
if err != nil {
logger.Warn("Failed to retrieve EC2 hostname", zap.Error(err))
} else {
logger.Warn("Failed to get EC2 hostname", zap.Error(err))
defer metadataOutput.Content.Close()
hostnameBytes, readErr := io.ReadAll(metadataOutput.Content)
if readErr != nil {
logger.Warn("Failed to read EC2 hostname content", zap.Error(readErr))
} else {
hostInfo.EC2Hostname = string(hostnameBytes)
}
}

return
Expand Down Expand Up @@ -94,13 +109,13 @@ type Provider struct {
}

func NewProvider(logger *zap.Logger) (*Provider, error) {
sess, err := session.NewSession()
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
return nil, err
}
return &Provider{
logger: logger,
detector: ec2provider.NewProvider(sess),
detector: ec2provider.NewProvider(cfg),
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The datadogexporter depends on the NewProvider function from metadataproviders, and the resourcedetectionprocessor also relies on the same function.

}, nil
}

Expand Down Expand Up @@ -129,23 +144,20 @@ func (p *Provider) instanceTags(ctx context.Context) (*ec2.DescribeTagsOutput, e
// Similar to:
// - https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/39dbc1ac8/processor/resourcedetectionprocessor/internal/aws/ec2/ec2.go#L118-L151
// - https://github.com/DataDog/datadog-agent/blob/1b4afdd6a03e8fabcc169b924931b2bb8935dab9/pkg/util/ec2/ec2_tags.go#L104-L134
sess, err := session.NewSession(&aws.Config{
Region: aws.String(meta.Region),
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(meta.Region),
)
if err != nil {
return nil, fmt.Errorf("failed to build AWS session: %w", err)
return nil, fmt.Errorf("failed to load AWS config: %w", err)
}

svc := ec2.New(sess)
return svc.DescribeTagsWithContext(ctx,
&ec2.DescribeTagsInput{
Filters: []*ec2.Filter{{
Name: aws.String("resource-id"),
Values: []*string{
aws.String(meta.InstanceID),
},
}},
})
client := ec2.NewFromConfig(cfg)
return client.DescribeTags(ctx, &ec2.DescribeTagsInput{
Filters: []types.Filter{{
Name: aws.String("resource-id"),
Values: []string{meta.InstanceID},
}},
})
}

// clusterNameFromTags gets the AWS EC2 Cluster name from the tags on an EC2 instance.
Expand Down
10 changes: 6 additions & 4 deletions exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ package ec2
import (
"testing"

"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"

"github.com/stretchr/testify/assert"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -56,7 +58,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "missing cluster name tag",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
},
},
Expand All @@ -66,7 +68,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "cluster name tag only has the prefix",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
{Key: strp("kubernetes.io/cluster/"), Value: strp("some value")},
},
Expand All @@ -77,7 +79,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "cluster name is available",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
{Key: strp("kubernetes.io/cluster/myclustername"), Value: strp("some value")},
},
Expand Down
42 changes: 32 additions & 10 deletions internal/metadataproviders/aws/ec2/metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,59 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib

import (
"context"
"fmt"
"io"

"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
)

type Provider interface {
Get(ctx context.Context) (ec2metadata.EC2InstanceIdentityDocument, error)
Get(ctx context.Context) (imds.InstanceIdentityDocument, error)
Hostname(ctx context.Context) (string, error)
InstanceID(ctx context.Context) (string, error)
}

type metadataClient struct {
metadata *ec2metadata.EC2Metadata
client *imds.Client
}

var _ Provider = (*metadataClient)(nil)

func NewProvider(sess *session.Session) Provider {
func NewProvider(cfg aws.Config) Provider {
return &metadataClient{
metadata: ec2metadata.New(sess),
client: imds.NewFromConfig(cfg),
}
}

func (c *metadataClient) getMetadata(ctx context.Context, path string) (string, error) {
output, err := c.client.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
if err != nil {
return "", fmt.Errorf("failed to get %s from IMDS: %w", path, err)
}
defer output.Content.Close()

data, err := io.ReadAll(output.Content)
if err != nil {
return "", fmt.Errorf("failed to read %s response: %w", path, err)
}

return string(data), nil
}

func (c *metadataClient) InstanceID(ctx context.Context) (string, error) {
return c.metadata.GetMetadataWithContext(ctx, "instance-id")
return c.getMetadata(ctx, "instance-id")
}

func (c *metadataClient) Hostname(ctx context.Context) (string, error) {
return c.metadata.GetMetadataWithContext(ctx, "hostname")
return c.getMetadata(ctx, "hostname")
}

func (c *metadataClient) Get(ctx context.Context) (ec2metadata.EC2InstanceIdentityDocument, error) {
return c.metadata.GetInstanceIdentityDocumentWithContext(ctx)
func (c *metadataClient) Get(ctx context.Context) (imds.InstanceIdentityDocument, error) {
output, err := c.client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
if err != nil {
return imds.InstanceIdentityDocument{}, fmt.Errorf("failed to get instance identity document: %w", err)
}

return output.InstanceIdentityDocument, nil
}
Loading
Loading