Skip to content

Commit

Permalink
Migrate OpenSearch to AWS SDK v2 (gruntwork-io#789)
Browse files Browse the repository at this point in the history 
* chore(resources): migrating opensearch to aws sdk v2

Signed-off-by: Scott Crooks <[email protected]>

* test(resources): migrating opensearch to aws sdk v2

Signed-off-by: Scott Crooks <[email protected]>

---------

Signed-off-by: Scott Crooks <[email protected]>
  • Loading branch information
@sc250024
sc250024 authored Nov 20, 2024
1 parent 174aedb commit b6a4b37
Show file tree
Hide file tree
Showing 6 changed files with 71 additions and 59 deletions.
53 changes: 27 additions & 26 deletions aws/resources/opensearch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,14 @@ import (
"sync"
"time"

"github.com/gruntwork-io/cloud-nuke/util"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/opensearch"
"github.com/aws/aws-sdk-go-v2/service/opensearch/types"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/opensearchservice"
"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/cloud-nuke/logging"
"github.com/gruntwork-io/cloud-nuke/report"
"github.com/gruntwork-io/cloud-nuke/util"
"github.com/gruntwork-io/go-commons/errors"
"github.com/gruntwork-io/go-commons/retry"
"github.com/hashicorp/go-multierror"
Expand Down Expand Up @@ -46,7 +47,7 @@ func (osd *OpenSearchDomains) getAll(c context.Context, configObj config.Config)
if firstSeenTime == nil {
err := osd.setFirstSeenTag(domain.ARN, time.Now().UTC())
if err != nil {
logging.Errorf("Error tagging the OpenSearch Domain with ARN %s with error: %s", aws.StringValue(domain.ARN), err.Error())
logging.Errorf("Error tagging the OpenSearch Domain with ARN %s with error: %s", aws.ToString(domain.ARN), err.Error())
return nil, errors.WithStackTrace(err)
}
}
Expand All @@ -63,9 +64,9 @@ func (osd *OpenSearchDomains) getAll(c context.Context, configObj config.Config)
}

// getAllActiveOpenSearchDomains filters all active OpenSearch domains, which are those that have the `Created` flag true and `Deleted` flag false.
func (osd *OpenSearchDomains) getAllActiveOpenSearchDomains() ([]*opensearchservice.DomainStatus, error) {
func (osd *OpenSearchDomains) getAllActiveOpenSearchDomains() ([]types.DomainStatus, error) {
allDomains := []*string{}
resp, err := osd.Client.ListDomainNamesWithContext(osd.Context, &opensearchservice.ListDomainNamesInput{})
resp, err := osd.Client.ListDomainNames(osd.Context, &opensearch.ListDomainNamesInput{})
if err != nil {
logging.Errorf("Error getting all OpenSearch domains")
return nil, errors.WithStackTrace(err)
Expand All @@ -74,16 +75,16 @@ func (osd *OpenSearchDomains) getAllActiveOpenSearchDomains() ([]*opensearchserv
allDomains = append(allDomains, domain.DomainName)
}

input := &opensearchservice.DescribeDomainsInput{DomainNames: allDomains}
describedDomains, describeErr := osd.Client.DescribeDomainsWithContext(osd.Context, input)
input := &opensearch.DescribeDomainsInput{DomainNames: aws.ToStringSlice(allDomains)}
describedDomains, describeErr := osd.Client.DescribeDomains(osd.Context, input)
if describeErr != nil {
logging.Errorf("Error describing Domains from input %s: ", input)
return nil, errors.WithStackTrace(describeErr)
}

filteredDomains := []*opensearchservice.DomainStatus{}
filteredDomains := []types.DomainStatus{}
for _, domain := range describedDomains.DomainStatusList {
if aws.BoolValue(domain.Created) && aws.BoolValue(domain.Deleted) == false {
if aws.ToBool(domain.Created) && aws.ToBool(domain.Deleted) == false {
filteredDomains = append(filteredDomains, domain)
}
}
Expand All @@ -92,20 +93,20 @@ func (osd *OpenSearchDomains) getAllActiveOpenSearchDomains() ([]*opensearchserv

// Tag an OpenSearch Domain identified by the given ARN when it's first seen by cloud-nuke
func (osd *OpenSearchDomains) setFirstSeenTag(domainARN *string, timestamp time.Time) error {
logging.Debugf("Tagging the OpenSearch Domain with ARN %s with first seen timestamp", aws.StringValue(domainARN))
logging.Debugf("Tagging the OpenSearch Domain with ARN %s with first seen timestamp", aws.ToString(domainARN))
firstSeenTime := util.FormatTimestamp(timestamp)

input := &opensearchservice.AddTagsInput{
input := &opensearch.AddTagsInput{
ARN: domainARN,
TagList: []*opensearchservice.Tag{
TagList: []types.Tag{
{
Key: aws.String(firstSeenTagKey),
Value: aws.String(firstSeenTime),
},
},
}

_, err := osd.Client.AddTags(input)
_, err := osd.Client.AddTags(osd.Context, input)
if err != nil {
return errors.WithStackTrace(err)
}
Expand All @@ -117,18 +118,18 @@ func (osd *OpenSearchDomains) setFirstSeenTag(domainARN *string, timestamp time.
func (osd *OpenSearchDomains) getFirstSeenTag(domainARN *string) (*time.Time, error) {
var firstSeenTime *time.Time

input := &opensearchservice.ListTagsInput{ARN: domainARN}
domainTags, err := osd.Client.ListTags(input)
input := &opensearch.ListTagsInput{ARN: domainARN}
domainTags, err := osd.Client.ListTags(osd.Context, input)
if err != nil {
logging.Errorf("Error getting the tags for OpenSearch Domain with ARN %s", aws.StringValue(domainARN))
logging.Errorf("Error getting the tags for OpenSearch Domain with ARN %s", aws.ToString(domainARN))
return firstSeenTime, errors.WithStackTrace(err)
}

for _, tag := range domainTags.TagList {
if util.IsFirstSeenTag(tag.Key) {
firstSeenTime, err := util.ParseTimestamp(tag.Value)
if err != nil {
logging.Errorf("Error parsing the `cloud-nuke-first-seen` tag for OpenSearch Domain with ARN %s", aws.StringValue(domainARN))
logging.Errorf("Error parsing the `cloud-nuke-first-seen` tag for OpenSearch Domain with ARN %s", aws.ToString(domainARN))
return firstSeenTime, errors.WithStackTrace(err)
}

Expand All @@ -139,7 +140,7 @@ func (osd *OpenSearchDomains) getFirstSeenTag(domainARN *string) (*time.Time, er
return firstSeenTime, nil
}

// nukeAll nukes the given list of OpenSearch domains concurrently. Note that the opensearchservice API
// nukeAll nukes the given list of OpenSearch domains concurrently. Note that the opensearch API
// does not support bulk delete, so this routine will spawn a goroutine for each domain that needs to be nuked so that
// they can be issued concurrently.
func (osd *OpenSearchDomains) nukeAll(identifiers []*string) error {
Expand All @@ -148,8 +149,8 @@ func (osd *OpenSearchDomains) nukeAll(identifiers []*string) error {
return nil
}

// NOTE: we don't need to do pagination here, because the pagination is handled by the caller to this function,
// based on OpenSearchDomains.MaxBatchSize, however we add a guard here to warn users when the batching fails and has a
// NOTE: we don't need to do pagination here, because the caller handles the pagination to this function,
// based on OpenSearchDomains.MaxBatchSize, however, we add a guard here to warn users when the batching fails and has a
// chance of throttling AWS. Since we concurrently make one call for each identifier, we pick 100 for the limit here
// because many APIs in AWS have a limit of 100 requests per second.
if len(identifiers) > 100 {
Expand Down Expand Up @@ -187,7 +188,7 @@ func (osd *OpenSearchDomains) nukeAll(identifiers []*string) error {
// Wait a maximum of 5 minutes: 10 seconds in between, up to 30 times
30, 10*time.Second,
func() error {
resp, err := osd.Client.DescribeDomainsWithContext(osd.Context, &opensearchservice.DescribeDomainsInput{DomainNames: identifiers})
resp, err := osd.Client.DescribeDomains(osd.Context, &opensearch.DescribeDomainsInput{DomainNames: aws.ToStringSlice(identifiers)})
if err != nil {
return errors.WithStackTrace(retry.FatalError{Underlying: err})
}
Expand All @@ -201,7 +202,7 @@ func (osd *OpenSearchDomains) nukeAll(identifiers []*string) error {
return errors.WithStackTrace(err)
}
for _, domainName := range identifiers {
logging.Debugf("[OK] OpenSearch Domain %s was deleted in %s", aws.StringValue(domainName), osd.Region)
logging.Debugf("[OK] OpenSearch Domain %s was deleted in %s", aws.ToString(domainName), osd.Region)
}
return nil
}
Expand All @@ -211,12 +212,12 @@ func (osd *OpenSearchDomains) nukeAll(identifiers []*string) error {
func (osd *OpenSearchDomains) deleteAsync(wg *sync.WaitGroup, errChan chan error, domainName *string) {
defer wg.Done()

input := &opensearchservice.DeleteDomainInput{DomainName: domainName}
_, err := osd.Client.DeleteDomainWithContext(osd.Context, input)
input := &opensearch.DeleteDomainInput{DomainName: domainName}
_, err := osd.Client.DeleteDomain(osd.Context, input)

// Record status of this resource
e := report.Entry{
Identifier: aws.StringValue(domainName),
Identifier: aws.ToString(domainName),
ResourceType: "OpenSearch Domain",
Error: err,
}
Expand Down
45 changes: 22 additions & 23 deletions aws/resources/opensearch_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,37 +6,36 @@ import (
"testing"
"time"

"github.com/aws/aws-sdk-go/aws"
awsgo "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/opensearchservice"
"github.com/aws/aws-sdk-go/service/opensearchservice/opensearchserviceiface"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/opensearch"
"github.com/aws/aws-sdk-go-v2/service/opensearch/types"

"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/cloud-nuke/util"
"github.com/stretchr/testify/require"
)

type mockedOpenSearch struct {
opensearchserviceiface.OpenSearchServiceAPI
ListDomainNamesOutput opensearchservice.ListDomainNamesOutput
DescribeDomainsOutput opensearchservice.DescribeDomainsOutput
ListTagsOutput opensearchservice.ListTagsOutput
DeleteDomainOutput opensearchservice.DeleteDomainOutput
OpenSearchDomainsAPI
ListDomainNamesOutput opensearch.ListDomainNamesOutput
DescribeDomainsOutput opensearch.DescribeDomainsOutput
ListTagsOutput opensearch.ListTagsOutput
DeleteDomainOutput opensearch.DeleteDomainOutput
}

func (m mockedOpenSearch) DeleteDomainWithContext(_ awsgo.Context, _ *opensearchservice.DeleteDomainInput, _ ...request.Option) (*opensearchservice.DeleteDomainOutput, error) {
func (m mockedOpenSearch) DeleteDomain(ctx context.Context, params *opensearch.DeleteDomainInput, optFns ...func(*opensearch.Options)) (*opensearch.DeleteDomainOutput, error) {
return &m.DeleteDomainOutput, nil
}

func (m mockedOpenSearch) ListDomainNamesWithContext(_ awsgo.Context, _ *opensearchservice.ListDomainNamesInput, _ ...request.Option) (*opensearchservice.ListDomainNamesOutput, error) {
func (m mockedOpenSearch) ListDomainNames(ctx context.Context, params *opensearch.ListDomainNamesInput, optFns ...func(*opensearch.Options)) (*opensearch.ListDomainNamesOutput, error) {
return &m.ListDomainNamesOutput, nil
}

func (m mockedOpenSearch) DescribeDomainsWithContext(_ awsgo.Context, _ *opensearchservice.DescribeDomainsInput, _ ...request.Option) (*opensearchservice.DescribeDomainsOutput, error) {
func (m mockedOpenSearch) DescribeDomains(ctx context.Context, params *opensearch.DescribeDomainsInput, optFns ...func(*opensearch.Options)) (*opensearch.DescribeDomainsOutput, error) {
return &m.DescribeDomainsOutput, nil
}

func (m mockedOpenSearch) ListTags(*opensearchservice.ListTagsInput) (*opensearchservice.ListTagsOutput, error) {
func (m mockedOpenSearch) ListTags(ctx context.Context, params *opensearch.ListTagsInput, optFns ...func(*opensearch.Options)) (*opensearch.ListTagsOutput, error) {
return &m.ListTagsOutput, nil
}

Expand All @@ -53,15 +52,15 @@ func TestOpenSearch_GetAll(t *testing.T) {
now := time.Now()
osd := OpenSearchDomains{
Client: mockedOpenSearch{
ListDomainNamesOutput: opensearchservice.ListDomainNamesOutput{
DomainNames: []*opensearchservice.DomainInfo{
ListDomainNamesOutput: opensearch.ListDomainNamesOutput{
DomainNames: []types.DomainInfo{
{DomainName: aws.String(testName1)},
{DomainName: aws.String(testName2)},
},
},

ListTagsOutput: opensearchservice.ListTagsOutput{
TagList: []*opensearchservice.Tag{
ListTagsOutput: opensearch.ListTagsOutput{
TagList: []types.Tag{
{
Key: aws.String(firstSeenTagKey),
Value: aws.String(util.FormatTimestamp(now)),
Expand All @@ -73,8 +72,8 @@ func TestOpenSearch_GetAll(t *testing.T) {
},
},

DescribeDomainsOutput: opensearchservice.DescribeDomainsOutput{
DomainStatusList: []*opensearchservice.DomainStatus{
DescribeDomainsOutput: opensearch.DescribeDomainsOutput{
DomainStatusList: []types.DomainStatus{
{
DomainName: aws.String(testName1),
Created: aws.Bool(true),
Expand Down Expand Up @@ -125,7 +124,7 @@ func TestOpenSearch_GetAll(t *testing.T) {
OpenSearchDomain: tc.configObj,
})
require.NoError(t, err)
require.Equal(t, tc.expected, aws.StringValueSlice(names))
require.Equal(t, tc.expected, aws.ToStringSlice(names))
})
}
}
Expand All @@ -136,8 +135,8 @@ func TestOpenSearch_NukeAll(t *testing.T) {

osd := OpenSearchDomains{
Client: mockedOpenSearch{
DeleteDomainOutput: opensearchservice.DeleteDomainOutput{},
DescribeDomainsOutput: opensearchservice.DescribeDomainsOutput{},
DeleteDomainOutput: opensearch.DeleteDomainOutput{},
DescribeDomainsOutput: opensearch.DescribeDomainsOutput{},
},
}

Expand Down
27 changes: 18 additions & 9 deletions aws/resources/opensearch_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,35 @@ package resources
import (
"context"

awsgo "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/opensearchservice"
"github.com/aws/aws-sdk-go/service/opensearchservice/opensearchserviceiface"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/opensearch"

"github.com/gruntwork-io/cloud-nuke/config"
"github.com/gruntwork-io/go-commons/errors"
)

type OpenSearchDomainsAPI interface {
AddTags(ctx context.Context, params *opensearch.AddTagsInput, optFns ...func(*opensearch.Options)) (*opensearch.AddTagsOutput, error)
DeleteDomain(ctx context.Context, params *opensearch.DeleteDomainInput, optFns ...func(*opensearch.Options)) (*opensearch.DeleteDomainOutput, error)
DescribeDomains(ctx context.Context, params *opensearch.DescribeDomainsInput, optFns ...func(*opensearch.Options)) (*opensearch.DescribeDomainsOutput, error)
ListDomainNames(ctx context.Context, params *opensearch.ListDomainNamesInput, optFns ...func(*opensearch.Options)) (*opensearch.ListDomainNamesOutput, error)
ListTags(ctx context.Context, params *opensearch.ListTagsInput, optFns ...func(*opensearch.Options)) (*opensearch.ListTagsOutput, error)
}

// OpenSearchDomains represents all OpenSearch domains found in a region
type OpenSearchDomains struct {
BaseAwsResource
Client opensearchserviceiface.OpenSearchServiceAPI
Client OpenSearchDomainsAPI
Region string
DomainNames []string
}

func (osd *OpenSearchDomains) Init(session *session.Session) {
osd.Client = opensearchservice.New(session)
func (osd *OpenSearchDomains) InitV2(cfg aws.Config) {
osd.Client = opensearch.NewFromConfig(cfg)
}

func (osd *OpenSearchDomains) IsUsingV2() bool { return true }

// ResourceName is the simple name of the aws resource
func (osd *OpenSearchDomains) ResourceName() string {
return "opensearchdomain"
Expand Down Expand Up @@ -51,13 +60,13 @@ func (osd *OpenSearchDomains) GetAndSetIdentifiers(c context.Context, configObj
return nil, err
}

osd.DomainNames = awsgo.StringValueSlice(identifiers)
osd.DomainNames = aws.ToStringSlice(identifiers)
return osd.DomainNames, nil
}

// Nuke nukes all OpenSearch domain resources
func (osd *OpenSearchDomains) Nuke(identifiers []string) error {
if err := osd.nukeAll(awsgo.StringSlice(identifiers)); err != nil {
if err := osd.nukeAll(aws.StringSlice(identifiers)); err != nil {
return errors.WithStackTrace(err)
}
return nil
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/kms v1.37.5
github.com/aws/aws-sdk-go-v2/service/lambda v1.66.0
github.com/aws/aws-sdk-go-v2/service/macie2 v1.43.5
github.com/aws/aws-sdk-go-v2/service/opensearch v1.44.0
github.com/aws/aws-sdk-go-v2/service/scheduler v1.12.5
github.com/aws/aws-sdk-go-v2/service/securityhub v1.54.6
github.com/aws/aws-sdk-go-v2/service/ses v1.28.4
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ github.com/aws/aws-sdk-go-v2/service/lambda v1.66.0 h1:jMqMB8t/xbJnDdr11kH5rKdAh
github.com/aws/aws-sdk-go-v2/service/lambda v1.66.0/go.mod h1:4L6vIpiChdahncljlDFzKWGiZsLgszGwDoYqMDhb6T4=
github.com/aws/aws-sdk-go-v2/service/macie2 v1.43.5 h1:WJZG9ajCzS93t5qgG3NqVJqJqvpab9frc+2OhEszF/s=
github.com/aws/aws-sdk-go-v2/service/macie2 v1.43.5/go.mod h1:uAeVP7yiKq1iECAGBsHgMzlUH77OIrmsA1CBmIIeCe4=
github.com/aws/aws-sdk-go-v2/service/opensearch v1.44.0 h1:5U5Y6tWzqoP2Dr9APxkElg3tdMBsZd6PVWAq6NMYBbs=
github.com/aws/aws-sdk-go-v2/service/opensearch v1.44.0/go.mod h1:JbyxgIAzR9wXnvVAqITjrpKRCcktIC+UWtPJ2meWZbg=
github.com/aws/aws-sdk-go-v2/service/scheduler v1.12.5 h1:iitjgZ+/+ICxhA4EBvT5sqKchDcTiKBSOy+Ow0N8M+U=
github.com/aws/aws-sdk-go-v2/service/scheduler v1.12.5/go.mod h1:xbnM3QuSlc52qQjTdDK3GptxYgDnGaonugLFdv5opq4=
github.com/aws/aws-sdk-go-v2/service/securityhub v1.54.6 h1:EXJOAcfyr0atxiQdrHTnrVTCwO9udESAcQ96sWiJ8es=
Expand Down
2 changes: 1 addition & 1 deletion v2_migration_report/output.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ run `go generate ./...` to refresh this report.
| network-firewall-tls-config | |
| network-interface | |
| oidcprovider | |
| opensearchdomain | |
| opensearchdomain | :white_check_mark: |
| rds | |
| rds-cluster | |
| rds-global-cluster | |
Expand Down

0 comments on commit b6a4b37

Please sign in to comment.