Skip to content
Permalink

Comparing changes

This is a direct comparison between two commits made in this repository or its related repositories. View the default comparison for this range or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: GoogleCloudPlatform/spanner-migration-tool
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: ed49c6597671c579db5fa9d0aaf8b224c5bd2189
Choose a base ref
..
head repository: GoogleCloudPlatform/spanner-migration-tool
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: 2e097591ca80ee966f9e60d478f768319f1b0ae6
Choose a head ref
Showing with 1,855 additions and 539 deletions.
  1. +1 −1 .github/workflows/integration-tests-against-emulator.yaml
  2. +9 −0 accessors/clients/spanner/admin/interface.go
  3. +5 −0 accessors/clients/spanner/admin/mocks.go
  4. +76 −0 accessors/clients/spanner/client/interface.go
  5. +46 −0 accessors/clients/spanner/client/mocks.go
  6. +9 −0 accessors/clients/spanner/client/spanner_client.go
  7. +45 −36 accessors/spanner/mocks.go
  8. +113 −49 accessors/spanner/spanner_accessor.go
  9. +66 −18 accessors/spanner/spanner_accessor_test.go
  10. +3 −5 cmd/data.go
  11. +19 −3 cmd/schema.go
  12. +22 −26 cmd/utils.go
  13. +5 −0 common/constants/constants.go
  14. +87 −0 common/parse/parse.go
  15. +94 −0 common/task/task.go
  16. +52 −0 common/task/task_test.go
  17. +3 −67 common/utils/utils.go
  18. +2 −1 conversion/conversion.go
  19. +14 −15 conversion/data_from_database.go
  20. +4 −3 conversion/mocks.go
  21. +12 −15 conversion/resource_generation.go
  22. +18 −22 conversion/resource_generation_test.go
  23. +10 −1 docs/cli/data.md
  24. +9 −0 docs/cli/schema-and-data.md
  25. +17 −7 docs/cli/schema.md
  26. +25 −0 docs/ui/schema-conv/spanner-draft.md
  27. +131 −0 expressions_api/expression_verify.go
  28. +299 −0 expressions_api/expression_verify_test.go
  29. +29 −0 internal/convert.go
  30. +4 −3 sources/common/infoschema.go
  31. +12 −10 sources/common/mocks.go
  32. +0 −70 sources/common/utils.go
  33. +0 −19 sources/common/utils_test.go
  34. +392 −0 test_data/session_expression_verify.json
  35. +3 −3 testing/accessors/spanner/spanner_accessor_test.go
  36. +10 −10 testing/conversion/conversion_test.go
  37. +114 −0 testing/expressions_api/integration_test.go
  38. +89 −142 ui/package-lock.json
  39. +2 −5 webv2/helpers/helpers.go
  40. +2 −3 webv2/routes.go
  41. +2 −5 webv2/session/session_service.go
2 changes: 1 addition & 1 deletion .github/workflows/integration-tests-against-emulator.yaml
Original file line number Diff line number Diff line change
@@ -52,7 +52,7 @@ jobs:

services:
spanner_emulator:
image: gcr.io/cloud-spanner-emulator/emulator:1.5.23
image: gcr.io/cloud-spanner-emulator/emulator:1.5.27
ports:
- 9010:9010
- 9020:9020
9 changes: 9 additions & 0 deletions accessors/clients/spanner/admin/interface.go
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@ type AdminClient interface {
CreateDatabase(ctx context.Context, req *databasepb.CreateDatabaseRequest, opts ...gax.CallOption) (CreateDatabaseOperation, error)
UpdateDatabaseDdl(ctx context.Context, req *databasepb.UpdateDatabaseDdlRequest, opts ...gax.CallOption) (UpdateDatabaseDdlOperation, error)
GetDatabaseDdl(ctx context.Context, req *databasepb.GetDatabaseDdlRequest, opts ...gax.CallOption) (*databasepb.GetDatabaseDdlResponse, error)
DropDatabase(ctx context.Context, req *databasepb.DropDatabaseRequest, opts ...gax.CallOption) (error)
}

// Use this interface instead of database.CreateDatabaseOperation to support mocking.
@@ -92,4 +93,12 @@ func (c *UpdateDatabaseDdlImpl) Wait(ctx context.Context, opts ...gax.CallOption

func (c *AdminClientImpl) GetDatabaseDdl(ctx context.Context, req *databasepb.GetDatabaseDdlRequest, opts ...gax.CallOption) (*databasepb.GetDatabaseDdlResponse, error) {
return c.adminClient.GetDatabaseDdl(ctx, req, opts...)
}

func (c *AdminClientImpl) DropDatabase(ctx context.Context, req *databasepb.DropDatabaseRequest, opts ...gax.CallOption) error {
err := c.adminClient.DropDatabase(ctx, req, opts...)
if err != nil {
return err
}
return nil
}
5 changes: 5 additions & 0 deletions accessors/clients/spanner/admin/mocks.go
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@ type AdminClientMock struct {
CreateDatabaseMock func(ctx context.Context, req *databasepb.CreateDatabaseRequest, opts ...gax.CallOption) (CreateDatabaseOperation, error)
UpdateDatabaseDdlMock func(ctx context.Context, req *databasepb.UpdateDatabaseDdlRequest, opts ...gax.CallOption) (UpdateDatabaseDdlOperation, error)
GetDatabaseDdlMock func(ctx context.Context, req *databasepb.GetDatabaseDdlRequest, opts ...gax.CallOption) (*databasepb.GetDatabaseDdlResponse, error)
DropDatabaseMock func(ctx context.Context, req *databasepb.DropDatabaseRequest, opts ...gax.CallOption) error
}

func (acm *AdminClientMock) GetDatabase(ctx context.Context, req *databasepb.GetDatabaseRequest, opts ...gax.CallOption) (*databasepb.Database, error) {
@@ -45,6 +46,10 @@ func (acm *AdminClientMock) GetDatabaseDdl(ctx context.Context, req *databasepb.
return acm.GetDatabaseDdlMock(ctx, req, opts...)
}

func (acm *AdminClientMock) DropDatabase(ctx context.Context, req *databasepb.DropDatabaseRequest, opts ...gax.CallOption) error {
return acm.DropDatabaseMock(ctx, req, opts...)
}

// Mock that implements the CreateDatabaseOperation interface.
// Pass in unit tests where CreateDatabaseOperation is an input parameter.
type CreateDatabaseOperationMock struct {
76 changes: 76 additions & 0 deletions accessors/clients/spanner/client/interface.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package spannerclient

import (
"context"

"cloud.google.com/go/spanner"
)


type SpannerClient interface {
Single() ReadOnlyTransaction
DatabaseName() string
Refresh(ctx context.Context, dbURI string) error
}

type ReadOnlyTransaction interface {
Query(ctx context.Context, stmt spanner.Statement) RowIterator
}

type RowIterator interface {
Next() (*spanner.Row, error)
Stop()
}


// This implements the SpannerClient interface. This is the primary implementation that should be used in all places other than tests.
type SpannerClientImpl struct {
spannerClient *spanner.Client
}

func NewSpannerClientImpl(ctx context.Context, dbURI string) (*SpannerClientImpl, error) {
c, err := GetOrCreateClient(ctx, dbURI)
if err != nil {
return nil, err
}
return &SpannerClientImpl{spannerClient: c}, nil
}

func (c *SpannerClientImpl) Refresh(ctx context.Context, dbURI string) error {
var err error
c.spannerClient, err = CreateClient(ctx, dbURI)
if err != nil {
return err
}
return nil
}

func (c *SpannerClientImpl) Single() ReadOnlyTransaction {
rotxn := c.spannerClient.ReadOnlyTransaction()
return &ReadOnlyTransactionImpl{rotxn: rotxn}
}

func (c *SpannerClientImpl) DatabaseName() string {
return c.spannerClient.DatabaseName()
}

type ReadOnlyTransactionImpl struct {
rotxn *spanner.ReadOnlyTransaction
}

func (ro *ReadOnlyTransactionImpl) Query(ctx context.Context, stmt spanner.Statement) RowIterator {
ri := ro.rotxn.Query(ctx, stmt)
return &RowIteratorImpl{ri: ri}
}

type RowIteratorImpl struct {
ri *spanner.RowIterator
}

func (ri *RowIteratorImpl) Next() (*spanner.Row, error) {
return ri.ri.Next()
}

func (ri *RowIteratorImpl) Stop() {
ri.ri.Stop()
}
46 changes: 46 additions & 0 deletions accessors/clients/spanner/client/mocks.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package spannerclient

import (
"context"

"cloud.google.com/go/spanner"
)

type SpannerClientMock struct {
SingleMock func() ReadOnlyTransaction
DatabaseNameMock func() string
RefreshMock func(ctx context.Context, dbURI string) error
}

func (scm SpannerClientMock) Refresh(ctx context.Context, dbURI string) error {
return scm.RefreshMock(ctx, dbURI)
}

type ReadOnlyTransactionMock struct {
QueryMock func(ctx context.Context, stmt spanner.Statement) RowIterator
}

type RowIteratorMock struct {
NextMock func() (*spanner.Row, error)
StopMock func()
}

func (scm SpannerClientMock) Single() ReadOnlyTransaction {
return scm.SingleMock()
}

func (scm SpannerClientMock) DatabaseName() string {
return scm.DatabaseNameMock()
}

func (rom ReadOnlyTransactionMock) Query(ctx context.Context, stmt spanner.Statement) RowIterator {
return rom.QueryMock(ctx, stmt)
}

func (rim RowIteratorMock) Next() (*spanner.Row, error) {
return rim.NextMock()
}

func (rim RowIteratorMock) Stop() {
rim.StopMock()
}
9 changes: 9 additions & 0 deletions accessors/clients/spanner/client/spanner_client.go
Original file line number Diff line number Diff line change
@@ -41,3 +41,12 @@ func GetOrCreateClient(ctx context.Context, dbURI string) (*sp.Client, error) {
}
return spannerClient, nil
}

func CreateClient(ctx context.Context, dbURI string) (*sp.Client, error) {
var err error
spannerClient, err = newClient(ctx, dbURI)
if err != nil {
return nil, fmt.Errorf("failed to create spanner database client: %v", err)
}
return spannerClient, nil
}
81 changes: 45 additions & 36 deletions accessors/spanner/mocks.go
Original file line number Diff line number Diff line change
@@ -16,43 +16,43 @@ package spanneraccessor
import (
"context"

spanneradmin "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/admin"
spinstanceadmin "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/instanceadmin"
"github.com/GoogleCloudPlatform/spanner-migration-tool/internal"
)

// Mock that implements the SpannerAccessor interface.
// Pass in unit tests where SpannerAccessor is an input parameter.
type SpannerAccessorMock struct {
GetDatabaseDialectMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) (string, error)
CheckExistingDbMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) (bool, error)
CreateEmptyDatabaseMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) error
GetSpannerLeaderLocationMock func(ctx context.Context, instanceClient spinstanceadmin.InstanceAdminClient, instanceURI string) (string, error)
CheckIfChangeStreamExistsMock func(ctx context.Context, changeStreamName, dbURI string) (bool, error)
ValidateChangeStreamOptionsMock func(ctx context.Context, changeStreamName, dbURI string) error
CreateChangeStreamMock func(ctx context.Context, adminClient spanneradmin.AdminClient, changeStreamName, dbURI string) error
CreateDatabaseMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string, conv *internal.Conv, driver string, migrationType string) error
UpdateDatabaseMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string, conv *internal.Conv, driver string) error
CreateOrUpdateDatabaseMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI, driver string, conv *internal.Conv, migrationType string) error
VerifyDbMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) (dbExists bool, err error)
ValidateDDLMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) error
UpdateDDLForeignKeysMock func(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string, conv *internal.Conv, driver string, migrationType string)
GetDatabaseDialectMock func(ctx context.Context, dbURI string) (string, error)
CheckExistingDbMock func(ctx context.Context, dbURI string) (bool, error)
CreateEmptyDatabaseMock func(ctx context.Context, dbURI string) error
GetSpannerLeaderLocationMock func(ctx context.Context, instanceURI string) (string, error)
CheckIfChangeStreamExistsMock func(ctx context.Context, changeStreamName, dbURI string) (bool, error)
ValidateChangeStreamOptionsMock func(ctx context.Context, changeStreamName, dbURI string) error
CreateChangeStreamMock func(ctx context.Context, changeStreamName, dbURI string) error
CreateDatabaseMock func(ctx context.Context, dbURI string, conv *internal.Conv, driver string, migrationType string) error
UpdateDatabaseMock func(ctx context.Context, dbURI string, conv *internal.Conv, driver string) error
CreateOrUpdateDatabaseMock func(ctx context.Context, dbURI, driver string, conv *internal.Conv, migrationType string) error
VerifyDbMock func(ctx context.Context, dbURI string) (dbExists bool, err error)
ValidateDDLMock func(ctx context.Context, dbURI string) error
UpdateDDLForeignKeysMock func(ctx context.Context, dbURI string, conv *internal.Conv, driver string, migrationType string)
DropDatabaseMock func(ctx context.Context, dbURI string) error
ValidateDMLMock func(ctx context.Context, query string) (bool, error)
}

func (sam *SpannerAccessorMock) GetDatabaseDialect(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) (string, error) {
return sam.GetDatabaseDialectMock(ctx, adminClient, dbURI)
func (sam *SpannerAccessorMock) GetDatabaseDialect(ctx context.Context, dbURI string) (string, error) {
return sam.GetDatabaseDialectMock(ctx, dbURI)
}

func (sam *SpannerAccessorMock) CheckExistingDb(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) (bool, error) {
return sam.CheckExistingDbMock(ctx, adminClient, dbURI)
func (sam *SpannerAccessorMock) CheckExistingDb(ctx context.Context, dbURI string) (bool, error) {
return sam.CheckExistingDbMock(ctx, dbURI)
}

func (sam *SpannerAccessorMock) CreateEmptyDatabase(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) error {
return sam.CreateEmptyDatabaseMock(ctx, adminClient, dbURI)
func (sam *SpannerAccessorMock) CreateEmptyDatabase(ctx context.Context, dbURI string) error {
return sam.CreateEmptyDatabaseMock(ctx, dbURI)
}

func (sam *SpannerAccessorMock) GetSpannerLeaderLocation(ctx context.Context, instanceClient spinstanceadmin.InstanceAdminClient, instanceURI string) (string, error) {
return sam.GetSpannerLeaderLocationMock(ctx, instanceClient, instanceURI)
func (sam *SpannerAccessorMock) GetSpannerLeaderLocation(ctx context.Context, instanceURI string) (string, error) {
return sam.GetSpannerLeaderLocationMock(ctx, instanceURI)
}

func (sam *SpannerAccessorMock) CheckIfChangeStreamExists(ctx context.Context, changeStreamName, dbURI string) (bool, error) {
@@ -63,23 +63,32 @@ func (sam *SpannerAccessorMock) ValidateChangeStreamOptions(ctx context.Context,
return sam.ValidateChangeStreamOptionsMock(ctx, changeStreamName, dbURI)
}

func (sam *SpannerAccessorMock) CreateChangeStream(ctx context.Context, adminClient spanneradmin.AdminClient, changeStreamName, dbURI string) error {
return sam.CreateChangeStreamMock(ctx, adminClient, changeStreamName, dbURI)
func (sam *SpannerAccessorMock) CreateChangeStream(ctx context.Context, changeStreamName, dbURI string) error {
return sam.CreateChangeStreamMock(ctx, changeStreamName, dbURI)
}

func (sam *SpannerAccessorMock) CreateDatabase(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string, conv *internal.Conv, driver string, migrationType string) error {
return sam.CreateDatabaseMock(ctx,adminClient, dbURI, conv, driver, migrationType)
func (sam *SpannerAccessorMock) CreateDatabase(ctx context.Context, dbURI string, conv *internal.Conv, driver string, migrationType string) error {
return sam.CreateDatabaseMock(ctx, dbURI, conv, driver, migrationType)
}
func (sam *SpannerAccessorMock) UpdateDatabase(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string, conv *internal.Conv, driver string) error{
return sam.UpdateDatabaseMock(ctx, adminClient, dbURI, conv, driver)
func (sam *SpannerAccessorMock) UpdateDatabase(ctx context.Context, dbURI string, conv *internal.Conv, driver string) error {
return sam.UpdateDatabaseMock(ctx, dbURI, conv, driver)
}
func (sam *SpannerAccessorMock) CreateOrUpdateDatabase(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI, driver string, conv *internal.Conv, migrationType string) error {
return sam.CreateOrUpdateDatabaseMock(ctx,adminClient, dbURI, driver, conv, migrationType)
func (sam *SpannerAccessorMock) CreateOrUpdateDatabase(ctx context.Context, dbURI, driver string, conv *internal.Conv, migrationType string) error {
return sam.CreateOrUpdateDatabaseMock(ctx, dbURI, driver, conv, migrationType)
}
func (sam *SpannerAccessorMock) VerifyDb(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) (dbExists bool, err error){
return sam.VerifyDbMock(ctx, adminClient, dbURI)
func (sam *SpannerAccessorMock) VerifyDb(ctx context.Context, dbURI string) (dbExists bool, err error) {
return sam.VerifyDbMock(ctx, dbURI)
}
func (sam *SpannerAccessorMock) ValidateDDL(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string) error{
return sam.ValidateDDLMock(ctx, adminClient, dbURI)
func (sam *SpannerAccessorMock) ValidateDDL(ctx context.Context, dbURI string) error {
return sam.ValidateDDLMock(ctx, dbURI)
}
func (sam *SpannerAccessorMock) UpdateDDLForeignKeys(ctx context.Context, adminClient spanneradmin.AdminClient, dbURI string, conv *internal.Conv, driver string, migrationType string) {}
func (sam *SpannerAccessorMock) UpdateDDLForeignKeys(ctx context.Context, dbURI string, conv *internal.Conv, driver string, migrationType string) {
}
// DropDatabase implements SpannerAccessor.
func (sam *SpannerAccessorMock) DropDatabase(ctx context.Context, dbURI string) error {
return sam.DropDatabaseMock(ctx, dbURI)
}
// ValidateDML implements SpannerAccessor.
func (sam *SpannerAccessorMock) ValidateDML(ctx context.Context, query string) (bool, error) {
return sam.ValidateDMLMock(ctx, query)
}
Loading