Skip to content

Commit

Permalink
Merge pull request #112 from ubirch/UC-1606-db-migration-script-suppo…
Browse files Browse the repository at this point in the history 
…rt-v2

UC-1606 db migration script support
  • Loading branch information
@leroxyl
leroxyl authored Jan 20, 2023
2 parents b1b898d + 4d62f6f commit 9382219
Show file tree
Hide file tree
Showing 24 changed files with 2,412 additions and 881 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,4 @@ EXPOSE 8080/tcp
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=builder app/main/main ubirch-client
ENTRYPOINT ["/ubirch-client"]
CMD ["/data"]
CMD ["-configdirectory", "/data"]
93 changes: 40 additions & 53 deletions main/adapters/repository/database.go → main/adapters/database/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package repository
package database

import (
"context"
Expand All @@ -23,6 +23,7 @@ import (

"github.com/google/uuid"
"github.com/lib/pq"
"github.com/ubirch/ubirch-client-go/main/adapters/repository"
"github.com/ubirch/ubirch-client-go/main/ent"
"modernc.org/sqlite"

Expand Down Expand Up @@ -53,7 +54,7 @@ type DatabaseManager struct {
}

// Ensure Database implements the ContextManager interface
var _ ContextManager = (*DatabaseManager)(nil)
var _ repository.ContextManager = (*DatabaseManager)(nil)

// NewDatabaseManager takes a database connection string, returns a new initialized
// SQL database manager.
Expand All @@ -62,16 +63,19 @@ func NewDatabaseManager(driverName, dataSourceName string, maxConns int) (*Datab
return nil, fmt.Errorf("empty database driverName or dataSourceName")
}

var isolationLvl sql.IsolationLevel
var createStatements []string
dm := &DatabaseManager{}

switch driverName {
case PostgreSQL:
isolationLvl = sql.LevelReadCommitted
createStatements = createPostgres
dm.driverName = PostgreSQL
dm.options = &sql.TxOptions{
Isolation: sql.LevelReadCommitted,
}
case SQLite:
isolationLvl = sql.LevelSerializable
createStatements = createSQLite
dm.driverName = SQLite
dm.options = &sql.TxOptions{
Isolation: sql.LevelSerializable,
}
if !strings.Contains(dataSourceName, "?") {
dataSourceName += sqliteConfig
}
Expand All @@ -82,37 +86,26 @@ func NewDatabaseManager(driverName, dataSourceName string, maxConns int) (*Datab

log.Infof("initializing %s database connection", driverName)

db, err := sql.Open(driverName, dataSourceName)
var err error

dm.db, err = sql.Open(driverName, dataSourceName)
if err != nil {
return nil, err
}

db.SetMaxOpenConns(maxConns)
db.SetMaxIdleConns(maxConns)
db.SetConnMaxLifetime(10 * time.Minute)
db.SetConnMaxIdleTime(1 * time.Minute)

dm := &DatabaseManager{
options: &sql.TxOptions{
Isolation: isolationLvl,
ReadOnly: false,
},
db: db,
driverName: driverName,
dm.db.SetMaxOpenConns(maxConns)
dm.db.SetMaxIdleConns(maxConns)
dm.db.SetConnMaxLifetime(10 * time.Minute)
dm.db.SetConnMaxIdleTime(1 * time.Minute)

if err = dm.db.Ping(); err != nil {
return nil, err
}

if err = db.Ping(); err != nil {
if driverName == PostgreSQL && strings.Contains(err.Error(), "connection refused") {
// if there is no connection to the database yet, continue anyway.
log.Warnf("connection to the database could not yet be established: %v", err)
} else {
return nil, err
}
} else {
err = dm.CreateTables(createStatements)
if err != nil {
return nil, fmt.Errorf("creating DB table failed: %v", err)
}
// migrate database schema to the latest version
err = migrateUp(dm.db, dm.driverName)
if err != nil {
return nil, err
}

return dm, nil
Expand All @@ -133,15 +126,15 @@ func (dm *DatabaseManager) IsReady() error {
return nil
}

func (dm *DatabaseManager) StartTransaction(ctx context.Context) (transactionCtx TransactionCtx, err error) {
func (dm *DatabaseManager) StartTransaction(ctx context.Context) (transactionCtx repository.TransactionCtx, err error) {
err = dm.retry(func() error {
transactionCtx, err = dm.db.BeginTx(ctx, dm.options)
return err
})
return transactionCtx, err
}

func (dm *DatabaseManager) StoreIdentity(transactionCtx TransactionCtx, i ent.Identity) error {
func (dm *DatabaseManager) StoreIdentity(transactionCtx repository.TransactionCtx, i ent.Identity) error {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -163,15 +156,15 @@ func (dm *DatabaseManager) LoadIdentity(uid uuid.UUID) (*ent.Identity, error) {
err := dm.retry(func() error {
err := dm.db.QueryRow(query, uid).Scan(&i.PrivateKey, &i.PublicKey, &i.Signature, &i.AuthToken)
if err == sql.ErrNoRows {
return ErrNotExist
return repository.ErrNotExist
}
return err
})

return &i, err
}

func (dm *DatabaseManager) StoreActiveFlag(transactionCtx TransactionCtx, uid uuid.UUID, active bool) error {
func (dm *DatabaseManager) StoreActiveFlag(transactionCtx repository.TransactionCtx, uid uuid.UUID, active bool) error {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -184,7 +177,7 @@ func (dm *DatabaseManager) StoreActiveFlag(transactionCtx TransactionCtx, uid uu
return err
}

func (dm *DatabaseManager) LoadActiveFlagForUpdate(transactionCtx TransactionCtx, uid uuid.UUID) (active bool, err error) {
func (dm *DatabaseManager) LoadActiveFlagForUpdate(transactionCtx repository.TransactionCtx, uid uuid.UUID) (active bool, err error) {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return false, fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -199,7 +192,7 @@ func (dm *DatabaseManager) LoadActiveFlagForUpdate(transactionCtx TransactionCtx

err = tx.QueryRow(query, uid).Scan(&active)
if err == sql.ErrNoRows {
return false, ErrNotExist
return false, repository.ErrNotExist
}

return active, err
Expand All @@ -211,15 +204,15 @@ func (dm *DatabaseManager) LoadActiveFlag(uid uuid.UUID) (active bool, err error
err = dm.retry(func() error {
err := dm.db.QueryRow(query, uid).Scan(&active)
if err == sql.ErrNoRows {
return ErrNotExist
return repository.ErrNotExist
}
return err
})

return active, err
}

func (dm *DatabaseManager) StoreSignature(transactionCtx TransactionCtx, uid uuid.UUID, signature []byte) error {
func (dm *DatabaseManager) StoreSignature(transactionCtx repository.TransactionCtx, uid uuid.UUID, signature []byte) error {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -232,7 +225,7 @@ func (dm *DatabaseManager) StoreSignature(transactionCtx TransactionCtx, uid uui
return err
}

func (dm *DatabaseManager) LoadSignatureForUpdate(transactionCtx TransactionCtx, uid uuid.UUID) (signature []byte, err error) {
func (dm *DatabaseManager) LoadSignatureForUpdate(transactionCtx repository.TransactionCtx, uid uuid.UUID) (signature []byte, err error) {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return nil, fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -247,13 +240,13 @@ func (dm *DatabaseManager) LoadSignatureForUpdate(transactionCtx TransactionCtx,

err = tx.QueryRow(query, uid).Scan(&signature)
if err == sql.ErrNoRows {
return nil, ErrNotExist
return nil, repository.ErrNotExist
}

return signature, err
}

func (dm *DatabaseManager) StoreAuth(transactionCtx TransactionCtx, uid uuid.UUID, auth string) error {
func (dm *DatabaseManager) StoreAuth(transactionCtx repository.TransactionCtx, uid uuid.UUID, auth string) error {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -266,7 +259,7 @@ func (dm *DatabaseManager) StoreAuth(transactionCtx TransactionCtx, uid uuid.UUI
return err
}

func (dm *DatabaseManager) LoadAuthForUpdate(transactionCtx TransactionCtx, uid uuid.UUID) (auth string, err error) {
func (dm *DatabaseManager) LoadAuthForUpdate(transactionCtx repository.TransactionCtx, uid uuid.UUID) (auth string, err error) {
tx, ok := transactionCtx.(*sql.Tx)
if !ok {
return "", fmt.Errorf("transactionCtx for database manager is not of expected type *sql.Tx")
Expand All @@ -281,7 +274,7 @@ func (dm *DatabaseManager) LoadAuthForUpdate(transactionCtx TransactionCtx, uid

err = tx.QueryRow(query, uid).Scan(&auth)
if err == sql.ErrNoRows {
return "", ErrNotExist
return "", repository.ErrNotExist
}

return auth, err
Expand All @@ -306,7 +299,7 @@ func (dm *DatabaseManager) LoadExternalIdentity(ctx context.Context, uid uuid.UU
err := dm.retry(func() error {
err := dm.db.QueryRowContext(ctx, query, uid).Scan(&extId.PublicKey)
if err == sql.ErrNoRows {
return ErrNotExist
return repository.ErrNotExist
}
return err
})
Expand Down Expand Up @@ -369,12 +362,6 @@ func (dm *DatabaseManager) isRecoverable(err error) bool {
case "55P03", "53300", "53400": // lock_not_available, too_many_connections, configuration_limit_exceeded
time.Sleep(10 * time.Millisecond)
return true
case "42P01": // undefined_table
err = dm.CreateTables(createPostgres)
if err != nil {
log.Errorf("creating DB table failed: %v", err)
}
return true
}
log.Errorf("unexpected postgres database error: %s", pqErr)
}
Expand Down
7 changes: 4 additions & 3 deletions ...ers/repository/database_benchmark_test.go → ...pters/database/database_benchmark_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package repository
package database

import (
"context"
Expand All @@ -8,6 +8,7 @@ import (
"testing"

"github.com/stretchr/testify/require"
"github.com/ubirch/ubirch-client-go/main/adapters/repository"
)

func BenchmarkPostgres(b *testing.B) {
Expand Down Expand Up @@ -107,7 +108,7 @@ func BenchmarkSQLite_config(b *testing.B) {
}
}

func storeTestIdentity(t require.TestingT, ctxManager ContextManager) {
func storeTestIdentity(t require.TestingT, ctxManager repository.ContextManager) {
testId := getTestIdentity()

ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -123,7 +124,7 @@ func storeTestIdentity(t require.TestingT, ctxManager ContextManager) {
require.NoError(t, err)
}

func updateSignature(t require.TestingT, ctxManager ContextManager) {
func updateSignature(t require.TestingT, ctxManager repository.ContextManager) {
testId := getTestIdentity()

_, err := ctxManager.LoadActiveFlag(testId.Uid)
Expand Down
28 changes: 16 additions & 12 deletions ...apters/repository/database_sqlite_test.go → ...adapters/database/database_sqlite_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package repository
package database

import (
"context"
Expand All @@ -12,11 +12,15 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/ubirch/ubirch-client-go/main/adapters/repository"
"github.com/ubirch/ubirch-client-go/main/ent"
"modernc.org/sqlite"
)

const testSQLiteDSN = "test.db"
const (
testSQLiteDSN = "test.db"
sqliteTestLoad = 10
)

func TestDatabaseManager_sqlite(t *testing.T) {
dm, err := initSQLiteDB(t, 0)
Expand All @@ -30,23 +34,23 @@ func TestDatabaseManager_sqlite(t *testing.T) {

// check not exists
_, err = dm.LoadIdentity(testIdentity.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

_, err = dm.LoadActiveFlag(testIdentity.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

tx, err := dm.StartTransaction(ctx)
require.NoError(t, err)
require.NotNil(t, tx)

_, err = dm.LoadActiveFlagForUpdate(tx, testIdentity.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

_, err = dm.LoadSignatureForUpdate(tx, testIdentity.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

_, err = dm.LoadAuthForUpdate(tx, testIdentity.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

err = tx.Rollback()
require.NoError(t, err)
Expand Down Expand Up @@ -322,7 +326,7 @@ func TestDatabaseManager_CancelTransaction_sqlite(t *testing.T) {

// check transaction was rolled back
_, err = dm.LoadIdentity(testIdentity.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

// make sure identity can be stored now
ctx, cancel = context.WithCancel(context.Background())
Expand Down Expand Up @@ -406,7 +410,7 @@ func TestDatabaseManager_InvalidTransactionCtx_sqlite(t *testing.T) {
defer cleanUpDB(t, dm)

i := ent.Identity{}
mockCtx := &mockTx{}
mockCtx := &repository.MockTx{}

err = dm.StoreIdentity(mockCtx, i)
assert.EqualError(t, err, "transactionCtx for database manager is not of expected type *sql.Tx")
Expand Down Expand Up @@ -439,7 +443,7 @@ func TestDatabaseLoad_sqlite(t *testing.T) {

// generate identities
var testIdentities []ent.Identity
for i := 0; i < testLoad/10; i++ {
for i := 0; i < sqliteTestLoad; i++ {
id := getTestIdentity()
id.Uid = uuid.New()
testIdentities = append(testIdentities, id)
Expand All @@ -463,7 +467,7 @@ func TestDatabaseLoad_sqlite(t *testing.T) {
for _, testId := range testIdentities {
wg.Add(1)
go func(id ent.Identity) {
err := checkIdentity(dm, id, dbCheckAuth, wg)
err := checkIdentity(dm, id, wg)
if err != nil {
t.Errorf("%s: %v", id.Uid, err)
}
Expand Down Expand Up @@ -513,7 +517,7 @@ func TestDatabaseManager_StoreExternalIdentity_sqlite(t *testing.T) {
defer cancel()

_, err = dm.LoadExternalIdentity(ctx, testExtId.Uid)
assert.Equal(t, ErrNotExist, err)
assert.Equal(t, repository.ErrNotExist, err)

err = dm.StoreExternalIdentity(ctx, testExtId)
require.NoError(t, err)
Expand Down
Loading

0 comments on commit 9382219

Please sign in to comment.