From ae5cec8e70ba16ca4729b0f5ad87e184036104c2 Mon Sep 17 00:00:00 2001 From: Nitzan Ohana <16689354+nitzano@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:24:33 +0300 Subject: [PATCH 1/2] chore: 32 alllow to anonymize csv files (init) (#332) * chore(general): added csv keyword * chore: added dbzar command * chore: add papaparse * chore: parse ent * chore: move * chore: move files * chore: add hooks * chore: move files * chore: move * chore: fix * chore: move get-processor * chore: fix tests locally * chore: move collections * chore: move get-collections * chore: rename * chore: change databases * chore: rename enum * chore: rename * chore: lint fix * chore: set debug for dev * chore: milestone --- .dev.env | 3 +- .github/workflows/test.yml | 2 + jest-mongodb-config.js | 10 +++ package.json | 2 + src/api/anon-column.ts | 4 +- src/api/anon-db.ts | 6 +- .../anon-col/helpers/get-processor.ts | 33 ------- .../anon-col/helpers/process-column.ts | 5 +- .../{anon-db-action.ts => anon-db.action.ts} | 6 +- ...{anon-db-command.ts => anon-db.command.ts} | 2 +- src/cli/commands/anon-db/process-db.ts | 2 +- .../commands/anon-file/anon-file.action.ts | 22 +++++ .../commands/anon-file/anon-file.command.ts | 9 ++ src/cli/dbzar.ts | 4 +- src/config/types.ts | 2 +- .../mariadb/mariadb-processor.ts | 8 +- .../mongo/__tests__/mongo-processor.spec.ts | 6 +- .../{ => databases}/mongo/mongo-processor.ts | 10 +-- .../__tests__/postgres-processor.spec.ts | 88 +++++++++++++++++++ .../__tests__/postgres-processor.spec.ts | 5 +- .../postgres}/postgres/postgres-processor.ts | 8 +- .../{ => databases}/types/collection.ts | 2 +- .../{ => databases}/utils/get-collections.ts | 6 +- src/processors/files/index.ts | 0 src/processors/get-db-processor.ts | 33 +++++++ .../get-connection-string-engine.spec.ts | 6 +- src/utils/get-connection-string-engine.ts | 12 +-- tsconfig.json | 5 +- yarn.lock | 8 ++ 29 files changed, 228 insertions(+), 81 deletions(-) create mode 100644 jest-mongodb-config.js delete mode 100644 src/cli/commands/anon-col/helpers/get-processor.ts rename src/cli/commands/anon-db/{anon-db-action.ts => anon-db.action.ts} (79%) rename src/cli/commands/anon-db/{anon-db-command.ts => anon-db.command.ts} (82%) create mode 100644 src/cli/commands/anon-file/anon-file.action.ts create mode 100644 src/cli/commands/anon-file/anon-file.command.ts rename src/processors/{ => databases}/mariadb/mariadb-processor.ts (86%) rename src/processors/{ => databases}/mongo/__tests__/mongo-processor.spec.ts (89%) rename src/processors/{ => databases}/mongo/mongo-processor.ts (79%) create mode 100644 src/processors/databases/postgres/__tests__/postgres-processor.spec.ts rename src/processors/{ => databases/postgres}/postgres/__tests__/postgres-processor.spec.ts (94%) rename src/processors/{ => databases/postgres}/postgres/postgres-processor.ts (85%) rename src/processors/{ => databases}/types/collection.ts (65%) rename src/processors/{ => databases}/utils/get-collections.ts (78%) create mode 100644 src/processors/files/index.ts create mode 100644 src/processors/get-db-processor.ts diff --git a/.dev.env b/.dev.env index feddda66..19a902a5 100644 --- a/.dev.env +++ b/.dev.env @@ -1,4 +1,5 @@ MONGO_DB_URI=mongodb://root:example@localhost POSTGRES_DB_URI=postgresql://postgres:example@localhost/test MARIADB_DB_URI=mariadb://example:example@127.0.0.1/test -NODE_ENV=development \ No newline at end of file +NODE_ENV=development +DEBUG="dbzar,dbzar:*" \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef65129b..a57d2346 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,10 +5,12 @@ on: branches: - master - beta + - "milestone-**" pull_request: branches: - master - beta + - "milestones-**" jobs: lint: diff --git a/jest-mongodb-config.js b/jest-mongodb-config.js new file mode 100644 index 00000000..2f3d9263 --- /dev/null +++ b/jest-mongodb-config.js @@ -0,0 +1,10 @@ +module.exports = { + mongodbMemoryServerOptions: { + binary: { + version: '6.0.6', + skipMD5: true, + }, + autoStart: false, + instance: {}, + }, +}; diff --git a/package.json b/package.json index 258def94..14cfda78 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "mongodb", "mariadb", "mysql", + "csv", "mock", "fake" ], @@ -93,6 +94,7 @@ "mysql": "^2.18.1", "node-emoji": "^2.1.0", "ora": "5.4.1", + "papaparse": "^5.4.1", "pg": "^8.7.1" }, "packageManager": "yarn@3.3.1" diff --git a/src/api/anon-column.ts b/src/api/anon-column.ts index 850bcaa8..68550b29 100644 --- a/src/api/anon-column.ts +++ b/src/api/anon-column.ts @@ -1,7 +1,7 @@ import {type Anonymizer} from '../anonymizers/types'; import {createAnonymizer} from '../anonymizers/utils/create-anonymizer'; -import {getProcessor} from '../cli/commands/anon-col/helpers/get-processor'; import {type Processor} from '../processors/base-processor/processor'; +import {getDatabaseProcessor} from '../processors/get-db-processor'; import {createLogger} from '../services/loggers/debug-logger'; import {type Provider} from '../types/types'; @@ -29,7 +29,7 @@ export async function anonColumn( throw new Error('No uri in configuration'); } - const processor: Processor | undefined = getProcessor(uri); + const processor: Processor | undefined = getDatabaseProcessor(uri); logger(`processing ${dbName}`); if (processor) { diff --git a/src/api/anon-db.ts b/src/api/anon-db.ts index dc65b1b0..f617b7b2 100644 --- a/src/api/anon-db.ts +++ b/src/api/anon-db.ts @@ -1,7 +1,7 @@ -import {getProcessor} from '../cli/commands/anon-col/helpers/get-processor'; import {type Config} from '../config/types'; import {type Processor} from '../processors/base-processor/processor'; -import {getCollections} from '../processors/utils/get-collections'; +import {getCollections} from '../processors/databases/utils/get-collections'; +import {getDatabaseProcessor} from '../processors/get-db-processor'; import {createLogger} from '../services/loggers/debug-logger'; const logger = createLogger(__filename); @@ -20,7 +20,7 @@ export async function anonDb(config: Config): Promise { throw new Error('No uri in configuration'); } - const processor: Processor | undefined = getProcessor(uri); + const processor: Processor | undefined = getDatabaseProcessor(uri); logger(`processing ${dbName}`); if (processor) { diff --git a/src/cli/commands/anon-col/helpers/get-processor.ts b/src/cli/commands/anon-col/helpers/get-processor.ts deleted file mode 100644 index 212dcebe..00000000 --- a/src/cli/commands/anon-col/helpers/get-processor.ts +++ /dev/null @@ -1,33 +0,0 @@ -import {EngineType} from '../../../../config/types'; -import {type Processor} from '../../../../processors/base-processor/processor'; -import {MariaDbProcessor} from '../../../../processors/mariadb/mariadb-processor'; -import {MongoProcessor} from '../../../../processors/mongo/mongo-processor'; -import {PostgresProcessor} from '../../../../processors/postgres/postgres-processor'; -import {getConnectionStringEngine} from '../../../../utils/get-connection-string-engine'; - -export function getProcessor( - connectionStringUri: string, -): Processor | undefined { - // Parse the engine - const engine = getConnectionStringEngine(connectionStringUri); - let processor: Processor | undefined; - - if (engine) { - switch (engine) { - case EngineType.Mongo: - processor = new MongoProcessor(connectionStringUri); - break; - case EngineType.PostGres: - processor = new PostgresProcessor(connectionStringUri); - break; - case EngineType.MariaDB: - case EngineType.MySQL: - processor = new MariaDbProcessor(connectionStringUri); - break; - default: - break; - } - } - - return processor; -} diff --git a/src/cli/commands/anon-col/helpers/process-column.ts b/src/cli/commands/anon-col/helpers/process-column.ts index 2a88785d..dfba74b1 100644 --- a/src/cli/commands/anon-col/helpers/process-column.ts +++ b/src/cli/commands/anon-col/helpers/process-column.ts @@ -1,7 +1,7 @@ import ora from 'ora'; import {type Anonymizer} from '../../../../anonymizers/types'; import {type Processor} from '../../../../processors/base-processor/processor'; -import {getProcessor} from './get-processor'; +import {getDatabaseProcessor} from '../../../../processors/get-db-processor'; import {isUserConfirmed} from './is-user-confirmed'; export async function processColumn( @@ -12,7 +12,8 @@ export async function processColumn( columnName: string, checkConfirm = true, ) { - const processor: Processor | undefined = getProcessor(connectionString); + const processor: Processor | undefined = + getDatabaseProcessor(connectionString); if (processor) { // Check confirm diff --git a/src/cli/commands/anon-db/anon-db-action.ts b/src/cli/commands/anon-db/anon-db.action.ts similarity index 79% rename from src/cli/commands/anon-db/anon-db-action.ts rename to src/cli/commands/anon-db/anon-db.action.ts index c5834520..570601cc 100644 --- a/src/cli/commands/anon-db/anon-db-action.ts +++ b/src/cli/commands/anon-db/anon-db.action.ts @@ -1,9 +1,9 @@ import {type Command} from 'commander'; import {type Config} from '../../../config/types'; import {type Processor} from '../../../processors/base-processor/processor'; -import {getCollections} from '../../../processors/utils/get-collections'; +import {getCollections} from '../../../processors/databases/utils/get-collections'; +import {getDatabaseProcessor} from '../../../processors/get-db-processor'; import {createLogger} from '../../../services/loggers/debug-logger'; -import {getProcessor} from '../anon-col/helpers/get-processor'; import {processDb} from './process-db'; import {loadDbzarConfig} from './utils/load-dbzar-config'; @@ -25,7 +25,7 @@ export async function anonDbAction(this: Command) { logger(`uri = ${uri}`); - const processor: Processor | undefined = getProcessor(uri); + const processor: Processor | undefined = getDatabaseProcessor(uri); const collections = getCollections(config); if (processor && collections) { diff --git a/src/cli/commands/anon-db/anon-db-command.ts b/src/cli/commands/anon-db/anon-db.command.ts similarity index 82% rename from src/cli/commands/anon-db/anon-db-command.ts rename to src/cli/commands/anon-db/anon-db.command.ts index 5ca0567f..83ee93fe 100644 --- a/src/cli/commands/anon-db/anon-db-command.ts +++ b/src/cli/commands/anon-db/anon-db.command.ts @@ -1,5 +1,5 @@ import {Command} from 'commander'; -import {anonDbAction} from './anon-db-action'; +import {anonDbAction} from './anon-db.action'; export const anonDbCommand = new Command('anon-db'); diff --git a/src/cli/commands/anon-db/process-db.ts b/src/cli/commands/anon-db/process-db.ts index b7d28883..f6d8a81e 100644 --- a/src/cli/commands/anon-db/process-db.ts +++ b/src/cli/commands/anon-db/process-db.ts @@ -3,7 +3,7 @@ import ora from 'ora'; import {providerEmoji} from '../../../anonymizers/consts/provider-emoji'; import {providerVerb} from '../../../anonymizers/consts/provider-verb'; import {type Processor} from '../../../processors/base-processor/processor'; -import {type Collection} from '../../../processors/types/collection'; +import {type Collection} from '../../../processors/databases/types/collection'; import {type ProviderType} from '../../../types/types'; async function processCollection( diff --git a/src/cli/commands/anon-file/anon-file.action.ts b/src/cli/commands/anon-file/anon-file.action.ts new file mode 100644 index 00000000..5ca240ea --- /dev/null +++ b/src/cli/commands/anon-file/anon-file.action.ts @@ -0,0 +1,22 @@ +import {extname} from 'path'; +import {type Command} from 'commander'; +import {createLogger} from '../../../services/loggers/debug-logger'; + +const logger = createLogger(__filename); + +export async function anonFileAction(this: Command) { + logger('processing file'); + + const [filePath] = this.args; + logger(`filePath = ${filePath}`); + + if (filePath) { + const extension = extname(filePath)?.slice(1).toLowerCase(); + + if (extension === 'csv') { + logger('parsing csv file'); + } + } else { + throw new Error('no file path'); + } +} diff --git a/src/cli/commands/anon-file/anon-file.command.ts b/src/cli/commands/anon-file/anon-file.command.ts new file mode 100644 index 00000000..e20ddc04 --- /dev/null +++ b/src/cli/commands/anon-file/anon-file.command.ts @@ -0,0 +1,9 @@ +import {Command} from 'commander'; +import {anonFileAction} from './anon-file.action'; + +export const anonFileCommand = new Command('anon-file'); + +anonFileCommand + .description('Anonymize a single file') + .argument('[filePath]', 'file path') + .action(anonFileAction); diff --git a/src/cli/dbzar.ts b/src/cli/dbzar.ts index 06382f31..97479d42 100644 --- a/src/cli/dbzar.ts +++ b/src/cli/dbzar.ts @@ -3,11 +3,13 @@ import process from 'node:process'; import {Command} from 'commander'; import {anonColCommand} from './commands/anon-col/anon-col'; -import {anonDbCommand} from './commands/anon-db/anon-db-command'; +import {anonDbCommand} from './commands/anon-db/anon-db.command'; +import {anonFileCommand} from './commands/anon-file/anon-file.command'; const program = new Command(); program.addCommand(anonColCommand); program.addCommand(anonDbCommand); +program.addCommand(anonFileCommand); program.parse(process.argv); diff --git a/src/config/types.ts b/src/config/types.ts index 2a2cce7f..64a3d91a 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -1,6 +1,6 @@ import {type Provider, type ProviderType} from '../types/types'; -export enum EngineType { +export enum DatabaseEngineType { PostGres = 'postgres', Mongo = 'mongo', MariaDB = 'mariadb', diff --git a/src/processors/mariadb/mariadb-processor.ts b/src/processors/databases/mariadb/mariadb-processor.ts similarity index 86% rename from src/processors/mariadb/mariadb-processor.ts rename to src/processors/databases/mariadb/mariadb-processor.ts index 80a0cea8..a8f050ee 100644 --- a/src/processors/mariadb/mariadb-processor.ts +++ b/src/processors/databases/mariadb/mariadb-processor.ts @@ -1,8 +1,8 @@ import knex, {type Knex} from 'knex'; -import {type Anonymizer} from '../../anonymizers/types'; -import {debugLogger} from '../../services/loggers/debug-logger'; -import {BaseProcessor} from '../base-processor/base-processor'; -import {type Processor} from '../base-processor/processor'; +import {type Anonymizer} from '../../../anonymizers/types'; +import {debugLogger} from '../../../services/loggers/debug-logger'; +import {BaseProcessor} from '../../base-processor/base-processor'; +import {type Processor} from '../../base-processor/processor'; const logger = debugLogger.extend('mariadb-processor'); diff --git a/src/processors/mongo/__tests__/mongo-processor.spec.ts b/src/processors/databases/mongo/__tests__/mongo-processor.spec.ts similarity index 89% rename from src/processors/mongo/__tests__/mongo-processor.spec.ts rename to src/processors/databases/mongo/__tests__/mongo-processor.spec.ts index fc18d88c..f9a37246 100644 --- a/src/processors/mongo/__tests__/mongo-processor.spec.ts +++ b/src/processors/databases/mongo/__tests__/mongo-processor.spec.ts @@ -1,7 +1,7 @@ import process from 'process'; -import {type Db, MongoClient, type MongoClientOptions} from 'mongodb'; -import {MaskAnonymizer} from '../../../anonymizers/mask/mask-anonymizer'; -import {type Anonymizer} from '../../../anonymizers/types'; +import {MongoClient, type Db, type MongoClientOptions} from 'mongodb'; +import {MaskAnonymizer} from '../../../../anonymizers/mask/mask-anonymizer'; +import {type Anonymizer} from '../../../../anonymizers/types'; import {MongoProcessor} from '../mongo-processor'; describe('mongo-processor', () => { diff --git a/src/processors/mongo/mongo-processor.ts b/src/processors/databases/mongo/mongo-processor.ts similarity index 79% rename from src/processors/mongo/mongo-processor.ts rename to src/processors/databases/mongo/mongo-processor.ts index a093652d..cfaf7ba0 100644 --- a/src/processors/mongo/mongo-processor.ts +++ b/src/processors/databases/mongo/mongo-processor.ts @@ -1,8 +1,8 @@ -import {type Db, MongoClient} from 'mongodb'; -import {type Anonymizer} from '../../anonymizers/types'; -import {debugLogger} from '../../services/loggers/debug-logger'; -import {BaseProcessor} from '../base-processor/base-processor'; -import {type Processor} from '../base-processor/processor'; +import {MongoClient, type Db} from 'mongodb'; +import {type Anonymizer} from '../../../anonymizers/types'; +import {debugLogger} from '../../../services/loggers/debug-logger'; +import {BaseProcessor} from '../../base-processor/base-processor'; +import {type Processor} from '../../base-processor/processor'; const logger = debugLogger.extend('mongo-processor'); export class MongoProcessor extends BaseProcessor implements Processor { diff --git a/src/processors/databases/postgres/__tests__/postgres-processor.spec.ts b/src/processors/databases/postgres/__tests__/postgres-processor.spec.ts new file mode 100644 index 00000000..c31c6890 --- /dev/null +++ b/src/processors/databases/postgres/__tests__/postgres-processor.spec.ts @@ -0,0 +1,88 @@ +import {type Knex} from 'knex'; +import {newDb} from 'pg-mem'; +import {MaskAnonymizer} from '../../../../anonymizers/mask/mask-anonymizer'; +import {type Anonymizer} from '../../../../anonymizers/types'; +import {PostgresProcessor} from '../postgres/postgres-processor'; + +describe('PostgresProcessor', () => { + let knex: Knex; + + beforeEach(async () => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + knex = newDb().adapters.createKnex(); + }); + + afterEach(async () => { + await knex.destroy(); + }); + + it('should process a single doc', async () => { + await knex.schema.createTable('users', (table) => { + table.increments('id'); + table.string('firstName'); + }); + + await knex('users').insert({firstName: 'John'}); + + const selectedRows1 = await knex('users').select('firstName'); + expect(selectedRows1[0].firstName).toBe('John'); + + const spy1 = jest + .spyOn(knex.client, 'destroy') + .mockImplementationOnce(async () => { + // Do nothing + }); + const spy2 = jest + .spyOn(PostgresProcessor.prototype as any, 'buildClient') + .mockImplementationOnce(() => knex); + + // Anonymize the db + const processor: PostgresProcessor = new PostgresProcessor( + 'postgresql://localhost', + ); + + const anonymizer: Anonymizer = new MaskAnonymizer(); + await processor.processColumn('users', 'firstName', anonymizer); + + spy1.mockRestore(); + spy2.mockRestore(); + + // Check again + const selectedRows2 = await knex('users').select('firstName'); + expect(selectedRows2[0].firstName).toBe('****'); + }); + + it('should process multiple rows', async () => { + await knex.schema.createTable('users', (table) => { + table.increments('id'); + table.string('firstName'); + }); + + await knex('users').insert([{firstName: 'test1'}, {firstName: 'test2'}]); + + const spy1 = jest + .spyOn(knex.client, 'destroy') + .mockImplementationOnce(async () => { + // Do nothing + }); + const spy2 = jest + .spyOn(PostgresProcessor.prototype as any, 'buildClient') + .mockImplementationOnce(() => knex); + + // Anonymize the db + const processor: PostgresProcessor = new PostgresProcessor( + 'postgresql://localhost', + ); + + const anonymizer: Anonymizer = new MaskAnonymizer(); + await processor.processColumn('users', 'firstName', anonymizer); + + spy1.mockRestore(); + spy2.mockRestore(); + + // Check again + const selectedRows2 = await knex('users').select('firstName'); + expect(selectedRows2[0].firstName).toBe('*****'); + expect(selectedRows2[1].firstName).toBe('*****'); + }); +}); diff --git a/src/processors/postgres/__tests__/postgres-processor.spec.ts b/src/processors/databases/postgres/postgres/__tests__/postgres-processor.spec.ts similarity index 94% rename from src/processors/postgres/__tests__/postgres-processor.spec.ts rename to src/processors/databases/postgres/postgres/__tests__/postgres-processor.spec.ts index 5d1189ba..58e5769e 100644 --- a/src/processors/postgres/__tests__/postgres-processor.spec.ts +++ b/src/processors/databases/postgres/postgres/__tests__/postgres-processor.spec.ts @@ -1,7 +1,8 @@ import {type Knex} from 'knex'; import {newDb} from 'pg-mem'; -import {MaskAnonymizer} from '../../../anonymizers/mask/mask-anonymizer'; -import {type Anonymizer} from '../../../anonymizers/types'; + +import {MaskAnonymizer} from '../../../../../anonymizers/mask/mask-anonymizer'; +import {type Anonymizer} from '../../../../../anonymizers/types'; import {PostgresProcessor} from '../postgres-processor'; describe('PostgresProcessor', () => { diff --git a/src/processors/postgres/postgres-processor.ts b/src/processors/databases/postgres/postgres/postgres-processor.ts similarity index 85% rename from src/processors/postgres/postgres-processor.ts rename to src/processors/databases/postgres/postgres/postgres-processor.ts index ec6243fa..a2e981cf 100644 --- a/src/processors/postgres/postgres-processor.ts +++ b/src/processors/databases/postgres/postgres/postgres-processor.ts @@ -1,8 +1,8 @@ import knex, {type Knex} from 'knex'; -import {type Anonymizer} from '../../anonymizers/types'; -import {debugLogger} from '../../services/loggers/debug-logger'; -import {BaseProcessor} from '../base-processor/base-processor'; -import {type Processor} from '../base-processor/processor'; +import {type Anonymizer} from '../../../../anonymizers/types'; +import {debugLogger} from '../../../../services/loggers/debug-logger'; +import {BaseProcessor} from '../../../base-processor/base-processor'; +import {type Processor} from '../../../base-processor/processor'; const logger = debugLogger.extend('postgress-processor'); diff --git a/src/processors/types/collection.ts b/src/processors/databases/types/collection.ts similarity index 65% rename from src/processors/types/collection.ts rename to src/processors/databases/types/collection.ts index a0d44663..2831fb2c 100644 --- a/src/processors/types/collection.ts +++ b/src/processors/databases/types/collection.ts @@ -1,4 +1,4 @@ -import {type Anonymizer} from '../../anonymizers/types'; +import {type Anonymizer} from '../../../anonymizers/types'; export type Collection = { dbName: string; diff --git a/src/processors/utils/get-collections.ts b/src/processors/databases/utils/get-collections.ts similarity index 78% rename from src/processors/utils/get-collections.ts rename to src/processors/databases/utils/get-collections.ts index 07d53366..fa1024e6 100644 --- a/src/processors/utils/get-collections.ts +++ b/src/processors/databases/utils/get-collections.ts @@ -1,6 +1,6 @@ -import {type Anonymizer} from '../../anonymizers/types'; -import {createAnonymizer} from '../../anonymizers/utils/create-anonymizer'; -import {type Config} from '../../config/types'; +import {type Anonymizer} from '../../../anonymizers/types'; +import {createAnonymizer} from '../../../anonymizers/utils/create-anonymizer'; +import {type Config} from '../../../config/types'; import {type Collection} from '../types/collection'; export function getCollections(config: Config): Collection[] { diff --git a/src/processors/files/index.ts b/src/processors/files/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/processors/get-db-processor.ts b/src/processors/get-db-processor.ts new file mode 100644 index 00000000..21cbc9b5 --- /dev/null +++ b/src/processors/get-db-processor.ts @@ -0,0 +1,33 @@ +import {DatabaseEngineType} from '../config/types'; +import {getConnectionStringEngine} from '../utils/get-connection-string-engine'; +import {type Processor} from './base-processor/processor'; +import {MariaDbProcessor} from './databases/mariadb/mariadb-processor'; +import {MongoProcessor} from './databases/mongo/mongo-processor'; +import {PostgresProcessor} from './databases/postgres/postgres/postgres-processor'; + +export function getDatabaseProcessor( + connectionStringUri: string, +): Processor | undefined { + // Parse the engine + const engine = getConnectionStringEngine(connectionStringUri); + let processor: Processor | undefined; + + if (engine) { + switch (engine) { + case DatabaseEngineType.Mongo: + processor = new MongoProcessor(connectionStringUri); + break; + case DatabaseEngineType.PostGres: + processor = new PostgresProcessor(connectionStringUri); + break; + case DatabaseEngineType.MariaDB: + case DatabaseEngineType.MySQL: + processor = new MariaDbProcessor(connectionStringUri); + break; + default: + break; + } + } + + return processor; +} diff --git a/src/utils/__tests__/get-connection-string-engine.spec.ts b/src/utils/__tests__/get-connection-string-engine.spec.ts index b37dec07..a6604d17 100644 --- a/src/utils/__tests__/get-connection-string-engine.spec.ts +++ b/src/utils/__tests__/get-connection-string-engine.spec.ts @@ -1,16 +1,16 @@ -import {EngineType} from '../../config/types'; +import {DatabaseEngineType} from '../../config/types'; import {getConnectionStringEngine} from '../get-connection-string-engine'; describe('getConnectionStringEngine', () => { it('should allow to parse postgres connection string ', () => { const engine = getConnectionStringEngine('postgresql://localhost/db1'); - expect(engine).toBe(EngineType.PostGres); + expect(engine).toBe(DatabaseEngineType.PostGres); }); it('should allow to parse mongo connection strings', () => { const engine = getConnectionStringEngine( 'mongodb://root:example@mongo:27017', ); - expect(engine).toBe(EngineType.Mongo); + expect(engine).toBe(DatabaseEngineType.Mongo); }); }); diff --git a/src/utils/get-connection-string-engine.ts b/src/utils/get-connection-string-engine.ts index 8492321a..0a009ee1 100644 --- a/src/utils/get-connection-string-engine.ts +++ b/src/utils/get-connection-string-engine.ts @@ -1,23 +1,23 @@ import {ConnectionString} from 'connection-string'; -import {EngineType} from '../config/types'; +import {DatabaseEngineType} from '../config/types'; import {debugLogger} from '../services/loggers/debug-logger'; export function getConnectionStringEngine( connectionStringUri: string, -): EngineType | undefined { +): DatabaseEngineType | undefined { if (connectionStringUri) { const {protocol} = new ConnectionString(connectionStringUri); if (protocol) { debugLogger(`protocol = ${protocol}`); switch (protocol) { case 'postgresql': - return EngineType.PostGres; + return DatabaseEngineType.PostGres; case 'mongodb': - return EngineType.Mongo; + return DatabaseEngineType.Mongo; case 'mariadb': - return EngineType.MariaDB; + return DatabaseEngineType.MariaDB; case 'mysql': - return EngineType.MySQL; + return DatabaseEngineType.MySQL; default: break; } diff --git a/tsconfig.json b/tsconfig.json index bd519088..ecf8e12a 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -27,7 +27,7 @@ "module": "commonjs" /* Specify what module code is generated. */, "rootDir": "./src" /* Specify the root folder within your source files. */, // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ - // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "baseUrl": "./src", /* Specify the base directory to resolve non-relative module names. */ // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ @@ -98,5 +98,6 @@ // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, - "include": ["src/"] + "include": ["src/"], + } diff --git a/yarn.lock b/yarn.lock index 929c47b7..47d47763 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4010,6 +4010,7 @@ __metadata: node-emoji: ^2.1.0 nodemon: ^2.0.15 ora: 5.4.1 + papaparse: ^5.4.1 pg: ^8.7.1 pg-mem: ^2.2.2 prettier: ^2.5.1 @@ -9019,6 +9020,13 @@ __metadata: languageName: node linkType: hard +"papaparse@npm:^5.4.1": + version: 5.4.1 + resolution: "papaparse@npm:5.4.1" + checksum: fc9e52f7158dca3517c229e3309065b1ab5da6c7194572fba4f31ff138bc43e3c91182cc40365cc828f97fe10d0aca416068fd731661058bea0f69ddb84a411a + languageName: node + linkType: hard + "parent-module@npm:^1.0.0": version: 1.0.1 resolution: "parent-module@npm:1.0.1" From 4c6ae77349f4ddb145974f87d3f121346aaae055 Mon Sep 17 00:00:00 2001 From: Nitzan Ohana <16689354+nitzano@users.noreply.github.com> Date: Fri, 22 Sep 2023 13:25:06 +0300 Subject: [PATCH 2/2] chore: init