Replace literal ids with referential ids

polytomic · Aug 25, 2023 · 83e79d9 · 83e79d9
1 parent 305f3ef
commit 83e79d9
Show file tree

Hide file tree

Showing 11 changed files with 217 additions and 127 deletions.
diff --git a/importer/bulk_syncs.go b/importer/bulk_syncs.go
@@ -24,12 +24,13 @@ var (
 type BulkSyncs struct {
 	c *polytomic.Client
 
-	Resources []polytomic.BulkSyncResponse
+	Resources map[string]polytomic.BulkSyncResponse
 }
 
 func NewBulkSyncs(c *polytomic.Client) *BulkSyncs {
 	return &BulkSyncs{
-		c: c,
+		c:         c,
+		Resources: make(map[string]polytomic.BulkSyncResponse),
 	}
 }
 
@@ -38,13 +39,18 @@ func (b *BulkSyncs) Init(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	b.Resources = bulkSyncs
+	for _, bulk := range bulkSyncs {
+		// Bulk sync names are not unique, so we need to a slug to the name
+		// to make it unique.
+		name := provider.ValidName(provider.ToSnakeCase(bulk.Name) + "_" + bulk.ID[:8])
+		b.Resources[name] = bulk
+	}
 
 	return nil
 }
 
-func (b *BulkSyncs) GenerateTerraformFiles(ctx context.Context, writer io.Writer) error {
-	for _, bulkSync := range b.Resources {
+func (b *BulkSyncs) GenerateTerraformFiles(ctx context.Context, writer io.Writer, refs map[string]string) error {
+	for name, bulkSync := range b.Resources {
 		bulkSchemas, err := b.c.Bulk().GetBulkSyncSchemas(ctx, bulkSync.ID)
 		if err != nil {
 			return err
@@ -57,9 +63,7 @@ func (b *BulkSyncs) GenerateTerraformFiles(ctx context.Context, writer io.Writer
 		}
 		hclFile := hclwrite.NewEmptyFile()
 		body := hclFile.Body()
-		// Bulk sync names are not unique, so we need to a slug to the name
-		// to make it unique.
-		name := provider.ValidName(provider.ToSnakeCase(bulkSync.Name) + "_" + bulkSync.ID[:8])
+
 		resourceBlock := body.AppendNewBlock("resource", []string{BulkSyncResource, name})
 		resourceBlock.Body().SetAttributeValue("name", cty.StringVal(bulkSync.Name))
 		resourceBlock.Body().SetAttributeValue("source_connection_id", cty.StringVal(bulkSync.SourceConnectionID))
@@ -79,17 +83,14 @@ func (b *BulkSyncs) GenerateTerraformFiles(ctx context.Context, writer io.Writer
 		resourceBlock.Body().SetAttributeValue("schedule", typeConverter(schedule))
 		body.AppendNewline()
 
-		writer.Write(hclFile.Bytes())
+		writer.Write(ReplaceRefs(hclFile.Bytes(), refs))
 	}
 
 	return nil
 }
 
 func (b *BulkSyncs) GenerateImports(ctx context.Context, writer io.Writer) error {
-	for _, bulkSync := range b.Resources {
-		// Bulk sync names are not unique, so we need to a slug to the name
-		// to make it unique.
-		name := provider.ValidName(provider.ToSnakeCase(bulkSync.Name) + "_" + bulkSync.ID[:8])
+	for name, bulkSync := range b.Resources {
 		writer.Write([]byte(fmt.Sprintf("terraform import %s.%s %s",
 			BulkSyncResource,
 			name,
@@ -102,3 +103,15 @@ func (b *BulkSyncs) GenerateImports(ctx context.Context, writer io.Writer) error
 func (b *BulkSyncs) Filename() string {
 	return BulkSyncResourceFileName
 }
+
+func (b *BulkSyncs) ResourceRefs() map[string]string {
+	result := make(map[string]string)
+	for name, bulk := range b.Resources {
+		result[bulk.ID] = name
+	}
+	return result
+}
+
+func (b *BulkSyncs) DatasourceRefs() map[string]string {
+	return nil
+}
diff --git a/importer/connections.go b/importer/connections.go
@@ -25,8 +25,8 @@ var (
 type Connections struct {
 	c *polytomic.Client
 
-	Resources   []Connection
-	Datasources []Connection
+	Resources   map[string]Connection
+	Datasources map[string]Connection
 }
 
 type Connection struct {
@@ -40,7 +40,9 @@ type Connection struct {
 
 func NewConnections(c *polytomic.Client) *Connections {
 	return &Connections{
-		c: c,
+		c:           c,
+		Resources:   make(map[string]Connection),
+		Datasources: make(map[string]Connection),
 	}
 }
 
@@ -50,55 +52,56 @@ func (c *Connections) Init(ctx context.Context) error {
 		return err
 	}
 	for _, conn := range conns {
+		name := provider.ValidName(provider.ToSnakeCase(conn.Name))
 		if r, ok := provider.ConnectionsMap[conn.Type.ID]; ok {
 			resp := &resource.MetadataResponse{}
 			r.Metadata(ctx, resource.MetadataRequest{
 				ProviderTypeName: provider.Name,
 			}, resp)
-			c.Resources = append(c.Resources, Connection{
+			c.Resources[name] = Connection{
 				ID:            conn.ID,
 				Resource:      resp.TypeName,
 				Name:          conn.Name,
 				Organization:  conn.OrganizationId,
 				Configuration: conn.Configuration,
-			})
+			}
 
 		} else if d, ok := provider.ConnectionDatasourcesMap[conn.Type.ID]; ok {
 			resp := &datasource.MetadataResponse{}
 			d.Metadata(ctx, datasource.MetadataRequest{
 				ProviderTypeName: provider.Name,
 			}, resp)
-			c.Datasources = append(c.Datasources, Connection{
+			c.Datasources[name] = Connection{
 				ID:           conn.ID,
 				Resource:     resp.TypeName,
 				Name:         conn.Name,
 				Organization: conn.OrganizationId,
-			})
+			}
 		} else {
 			log.Warn().Msgf("connection type %s not supported", conn.Type.ID)
 		}
 	}
 	return nil
 }
 
-func (c *Connections) GenerateTerraformFiles(ctx context.Context, writer io.Writer) error {
-	for _, conn := range c.Datasources {
+func (c *Connections) GenerateTerraformFiles(ctx context.Context, writer io.Writer, refs map[string]string) error {
+	for name, conn := range c.Datasources {
 		hclFile := hclwrite.NewEmptyFile()
 		body := hclFile.Body()
-		resourceBlock := body.AppendNewBlock("data", []string{conn.Resource, provider.ValidName(provider.ToSnakeCase(conn.Name))})
+		resourceBlock := body.AppendNewBlock("data", []string{conn.Resource, name})
 		resourceBlock.Body().SetAttributeValue("id", cty.StringVal(conn.ID))
 		resourceBlock.Body().SetAttributeValue("name", cty.StringVal(conn.Name))
 		resourceBlock.Body().SetAttributeValue("organization", cty.StringVal(conn.Organization))
 		body.AppendNewline()
 
-		writer.Write(hclFile.Bytes())
+		writer.Write(ReplaceRefs(hclFile.Bytes(), refs))
 	}
 
-	for _, conn := range c.Resources {
+	for name, conn := range c.Resources {
 		config := typeConverter(conn.Configuration)
 		hclFile := hclwrite.NewEmptyFile()
 		body := hclFile.Body()
-		resourceBlock := body.AppendNewBlock("resource", []string{conn.Resource, provider.ToSnakeCase(conn.Name)})
+		resourceBlock := body.AppendNewBlock("resource", []string{conn.Resource, name})
 		resourceBlock.Body().SetAttributeValue("name", cty.StringVal(conn.Name))
 		resourceBlock.Body().SetAttributeValue("organization", cty.StringVal(conn.Organization))
 		resourceBlock.Body().SetAttributeValue("configuration", config)
@@ -111,10 +114,10 @@ func (c *Connections) GenerateTerraformFiles(ctx context.Context, writer io.Writ
 }
 
 func (c *Connections) GenerateImports(ctx context.Context, writer io.Writer) error {
-	for _, conn := range c.Resources {
+	for name, conn := range c.Resources {
 		writer.Write([]byte(fmt.Sprintf("terraform import %s.%s %s",
 			conn.Resource,
-			provider.ValidName(provider.ToSnakeCase(conn.Name)),
+			name,
 			conn.ID)))
 		writer.Write([]byte(fmt.Sprintf(" # %s\n", conn.Name)))
 	}
@@ -124,3 +127,19 @@ func (c *Connections) GenerateImports(ctx context.Context, writer io.Writer) err
 func (c *Connections) Filename() string {
 	return ConnectionsResourceFileName
 }
+
+func (c *Connections) ResourceRefs() map[string]string {
+	result := make(map[string]string)
+	for name, conn := range c.Resources {
+		result[conn.ID] = fmt.Sprintf("%s.%s.id", conn.Resource, name)
+	}
+	return result
+}
+
+func (c *Connections) DatasourceRefs() map[string]string {
+	result := make(map[string]string)
+	for name, conn := range c.Datasources {
+		result[conn.ID] = fmt.Sprintf("%s.%s.id", conn.Resource, name)
+	}
+	return result
+}
diff --git a/importer/formatter.go b/importer/formatter.go
@@ -2,12 +2,26 @@ package importer
 
 import (
 	"fmt"
+	"regexp"
 
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/zclconf/go-cty/cty"
 	"golang.org/x/exp/slices"
 )
 
+// ReplaceRefs takes in an array of bytes and a map of references
+// It will parse the array of bytes and replace any reference byte sequences
+// with the value from the map.
+func ReplaceRefs(b []byte, refs map[string]string) []byte {
+	s := string(b)
+	for k, v := range refs {
+		// Repalce all instances of the reference and the enclosing quotes
+		re := regexp.MustCompile(`"` + k + `"`)
+		s = re.ReplaceAllString(s, v)
+	}
+	return []byte(s)
+}
+
 // convert arbitrary values to cty.Value
 func typeConverter(value any) cty.Value {
 	switch value := value.(type) {
@@ -209,3 +223,12 @@ func jsonEncodeMap(v map[string]any, wrapped ...string) hclwrite.Tokens {
 
 	return tokens
 }
+
+// referenceTokens takes in reference
+// and returns an unquoted value as hclwrite.Tokens
+func referenceTokens(ref string) hclwrite.Tokens {
+	return hclwrite.Tokens{
+		&hclwrite.Token{Bytes: []byte(" ")},
+		&hclwrite.Token{Bytes: []byte(ref)},
+	}
+}
diff --git a/importer/import.go b/importer/import.go
@@ -14,7 +14,9 @@ const (
 
 type Importable interface {
 	Init(ctx context.Context) error
-	GenerateTerraformFiles(ctx context.Context, writer io.Writer) error
+	ResourceRefs() map[string]string
+	DatasourceRefs() map[string]string
+	GenerateTerraformFiles(ctx context.Context, writer io.Writer, refs map[string]string) error
 	GenerateImports(ctx context.Context, writer io.Writer) error
 	Filename() string
 }
@@ -48,19 +50,28 @@ func Init(url, key, path string, recreate bool, includePermissions bool) {
 	}
 	defer importFile.Close()
 
+	refs := make(map[string]string)
 	for _, i := range importables {
 		err := i.Init(ctx)
 		if err != nil {
 			log.Fatal().AnErr("error", err).Msg("failed to initialize")
 		}
+		// Add resource refs
+		for k, v := range i.ResourceRefs() {
+			refs[k] = v
+		}
+		// Add datasource refs
+		for k, v := range i.DatasourceRefs() {
+			refs[k] = v
+		}
 		file := i.Filename()
 		f, err := createFile(recreate, 0644, path, file)
 		if err != nil {
 			log.Fatal().AnErr("error", err).Msg("failed to create file")
 		}
 		defer f.Close()
 
-		err = i.GenerateTerraformFiles(ctx, f)
+		err = i.GenerateTerraformFiles(ctx, f, refs)
 		if err != nil {
 			log.Fatal().AnErr("error", err).Msg("failed to generate terraform files")
 		}

diff --git a/importer/main.go b/importer/main.go
@@ -44,7 +44,7 @@ func (m *Main) Init(ctx context.Context) error {
 	return nil
 }
 
-func (m *Main) GenerateTerraformFiles(ctx context.Context, writer io.Writer) error {
+func (m *Main) GenerateTerraformFiles(ctx context.Context, writer io.Writer, refs map[string]string) error {
 	tmpl, err := template.New("main").Parse(mainTemplate)
 	if err != nil {
 		return err
@@ -71,3 +71,11 @@ func (m *Main) GenerateImports(ctx context.Context, writer io.Writer) error {
 func (m *Main) Filename() string {
 	return "main.tf"
 }
+
+func (m *Main) ResourceRefs() map[string]string {
+	return nil
+}
+
+func (m *Main) DatasourceRefs() map[string]string {
+	return nil
+}
diff --git a/importer/models.go b/importer/models.go
@@ -26,14 +26,15 @@ type Models struct {
 	modelNames  map[string]string
 	uniqueNames map[string]bool
 
-	Resources []*polytomic.Model
+	Resources map[string]*polytomic.Model
 }
 
 func NewModels(c *polytomic.Client) *Models {
 	return &Models{
 		c:           c,
 		modelNames:  map[string]string{},
 		uniqueNames: map[string]bool{},
+		Resources:   make(map[string]*polytomic.Model),
 	}
 }
 
@@ -49,17 +50,6 @@ func (m *Models) Init(ctx context.Context) error {
 			return err
 		}
 
-		m.Resources = append(m.Resources, hydratedModel)
-	}
-
-	return nil
-
-}
-
-func (m *Models) GenerateTerraformFiles(ctx context.Context, writer io.Writer) error {
-	for _, model := range m.Resources {
-		hclFile := hclwrite.NewEmptyFile()
-		body := hclFile.Body()
 		name := provider.ValidName(
 			provider.ToSnakeCase(model.Name),
 		)
@@ -68,6 +58,17 @@ func (m *Models) GenerateTerraformFiles(ctx context.Context, writer io.Writer) e
 		}
 		m.uniqueNames[name] = true
 		m.modelNames[model.ID] = name
+		m.Resources[name] = hydratedModel
+	}
+
+	return nil
+
+}
+
+func (m *Models) GenerateTerraformFiles(ctx context.Context, writer io.Writer, refs map[string]string) error {
+	for name, model := range m.Resources {
+		hclFile := hclwrite.NewEmptyFile()
+		body := hclFile.Body()
 
 		resourceBlock := body.AppendNewBlock("resource", []string{ModelResource, name})
 		resourceBlock.Body().SetAttributeValue("connection_id", cty.StringVal(model.ConnectionID))
@@ -114,7 +115,7 @@ func (m *Models) GenerateTerraformFiles(ctx context.Context, writer io.Writer) e
 		resourceBlock.Body().SetAttributeValue("identifier", cty.StringVal(model.Identifier))
 		resourceBlock.Body().SetAttributeValue("tracking_columns", typeConverter(model.TrackingColumns))
 
-		writer.Write(hclFile.Bytes())
+		writer.Write(ReplaceRefs(hclFile.Bytes(), refs))
 	}
 	return nil
 }
@@ -133,3 +134,15 @@ func (m *Models) GenerateImports(ctx context.Context, writer io.Writer) error {
 func (m *Models) Filename() string {
 	return ModelsResourceFileName
 }
+
+func (m *Models) ResourceRefs() map[string]string {
+	result := make(map[string]string)
+	for name, model := range m.Resources {
+		result[model.ID] = fmt.Sprintf("%s.%s.id", ModelResource, name)
+	}
+	return result
+}
+
+func (m *Models) DatasourceRefs() map[string]string {
+	return nil
+}