chore: 32 alllow to anonymize csv files (init) (#332)

* chore(general): added csv keyword

* chore: added dbzar command

* chore: add papaparse

* chore: parse ent

* chore: move

* chore: move files

* chore: add hooks

* chore: move files

* chore: move

* chore: fix

* chore: move get-processor

* chore: fix tests locally

* chore: move collections

* chore: move get-collections

* chore: rename

* chore: change databases

* chore: rename enum

* chore: rename

* chore: lint fix

* chore: set debug for dev

* chore: milestone

.dev.env

@@ -1,4 +1,5 @@
 MONGO_DB_URI=mongodb://root:example@localhost
 POSTGRES_DB_URI=postgresql://postgres:example@localhost/test
 MARIADB_DB_URI=mariadb://example:[email protected]/test
-NODE_ENV=development
+NODE_ENV=development
+DEBUG="dbzar,dbzar:*"

.github/workflows/test.yml

@@ -5,10 +5,12 @@ on:
     branches:
       - master
       - beta
+      - "milestone-**"
   pull_request:
     branches:
       - master
       - beta
+      - "milestones-**"
 
 jobs:
   lint:

jest-mongodb-config.js

@@ -0,0 +1,10 @@
+module.exports = {
+	mongodbMemoryServerOptions: {
+		binary: {
+			version: '6.0.6',
+			skipMD5: true,
+		},
+		autoStart: false,
+		instance: {},
+	},
+};

package.json

@@ -21,6 +21,7 @@
     "mongodb",
     "mariadb",
     "mysql",
+    "csv",
     "mock",
     "fake"
   ],
@@ -93,6 +94,7 @@
     "mysql": "^2.18.1",
     "node-emoji": "^2.1.0",
     "ora": "5.4.1",
+    "papaparse": "^5.4.1",
     "pg": "^8.7.1"
   },
   "packageManager": "[email protected]"

src/api/anon-column.ts

@@ -1,7 +1,7 @@
 import {type Anonymizer} from '../anonymizers/types';
 import {createAnonymizer} from '../anonymizers/utils/create-anonymizer';
-import {getProcessor} from '../cli/commands/anon-col/helpers/get-processor';
 import {type Processor} from '../processors/base-processor/processor';
+import {getDatabaseProcessor} from '../processors/get-db-processor';
 import {createLogger} from '../services/loggers/debug-logger';
 import {type Provider} from '../types/types';
 
@@ -29,7 +29,7 @@ export async function anonColumn(
 		throw new Error('No uri in configuration');
 	}
 
-	const processor: Processor | undefined = getProcessor(uri);
+	const processor: Processor | undefined = getDatabaseProcessor(uri);
 
 	logger(`processing ${dbName}`);
 	if (processor) {

src/api/anon-db.ts

@@ -1,7 +1,7 @@
-import {getProcessor} from '../cli/commands/anon-col/helpers/get-processor';
 import {type Config} from '../config/types';
 import {type Processor} from '../processors/base-processor/processor';
-import {getCollections} from '../processors/utils/get-collections';
+import {getCollections} from '../processors/databases/utils/get-collections';
+import {getDatabaseProcessor} from '../processors/get-db-processor';
 import {createLogger} from '../services/loggers/debug-logger';
 
 const logger = createLogger(__filename);
@@ -20,7 +20,7 @@ export async function anonDb(config: Config): Promise<void> {
 		throw new Error('No uri in configuration');
 	}
 
-	const processor: Processor | undefined = getProcessor(uri);
+	const processor: Processor | undefined = getDatabaseProcessor(uri);
 
 	logger(`processing ${dbName}`);
 	if (processor) {

src/cli/commands/anon-col/helpers/process-column.ts

@@ -1,7 +1,7 @@
 import ora from 'ora';
 import {type Anonymizer} from '../../../../anonymizers/types';
 import {type Processor} from '../../../../processors/base-processor/processor';
-import {getProcessor} from './get-processor';
+import {getDatabaseProcessor} from '../../../../processors/get-db-processor';
 import {isUserConfirmed} from './is-user-confirmed';
 
 export async function processColumn(
@@ -12,7 +12,8 @@ export async function processColumn(
 	columnName: string,
 	checkConfirm = true,
 ) {
-	const processor: Processor | undefined = getProcessor(connectionString);
+	const processor: Processor | undefined =
+		getDatabaseProcessor(connectionString);
 
 	if (processor) {
 		// Check confirm

src/cli/commands/anon-db/anon-db-action.ts → src/cli/commands/anon-db/anon-db.action.ts

@@ -1,9 +1,9 @@
 import {type Command} from 'commander';
 import {type Config} from '../../../config/types';
 import {type Processor} from '../../../processors/base-processor/processor';
-import {getCollections} from '../../../processors/utils/get-collections';
+import {getCollections} from '../../../processors/databases/utils/get-collections';
+import {getDatabaseProcessor} from '../../../processors/get-db-processor';
 import {createLogger} from '../../../services/loggers/debug-logger';
-import {getProcessor} from '../anon-col/helpers/get-processor';
 import {processDb} from './process-db';
 import {loadDbzarConfig} from './utils/load-dbzar-config';
 
@@ -25,7 +25,7 @@ export async function anonDbAction(this: Command) {
 
 	logger(`uri = ${uri}`);
 
-	const processor: Processor | undefined = getProcessor(uri);
+	const processor: Processor | undefined = getDatabaseProcessor(uri);
 	const collections = getCollections(config);
 
 	if (processor && collections) {

src/cli/commands/anon-db/anon-db-command.ts → src/cli/commands/anon-db/anon-db.command.ts

@@ -1,5 +1,5 @@
 import {Command} from 'commander';
-import {anonDbAction} from './anon-db-action';
+import {anonDbAction} from './anon-db.action';
 
 export const anonDbCommand = new Command('anon-db');
 

src/cli/commands/anon-db/process-db.ts

@@ -3,7 +3,7 @@ import ora from 'ora';
 import {providerEmoji} from '../../../anonymizers/consts/provider-emoji';
 import {providerVerb} from '../../../anonymizers/consts/provider-verb';
 import {type Processor} from '../../../processors/base-processor/processor';
-import {type Collection} from '../../../processors/types/collection';
+import {type Collection} from '../../../processors/databases/types/collection';
 import {type ProviderType} from '../../../types/types';
 
 async function processCollection(

src/cli/commands/anon-file/anon-file.action.ts

@@ -0,0 +1,22 @@
+import {extname} from 'path';
+import {type Command} from 'commander';
+import {createLogger} from '../../../services/loggers/debug-logger';
+
+const logger = createLogger(__filename);
+
+export async function anonFileAction(this: Command) {
+	logger('processing file');
+
+	const [filePath] = this.args;
+	logger(`filePath = ${filePath}`);
+
+	if (filePath) {
+		const extension = extname(filePath)?.slice(1).toLowerCase();
+
+		if (extension === 'csv') {
+			logger('parsing csv file');
+		}
+	} else {
+		throw new Error('no file path');
+	}
+}

src/cli/commands/anon-file/anon-file.command.ts

@@ -0,0 +1,9 @@
+import {Command} from 'commander';
+import {anonFileAction} from './anon-file.action';
+
+export const anonFileCommand = new Command('anon-file');
+
+anonFileCommand
+	.description('Anonymize a single file')
+	.argument('[filePath]', 'file path')
+	.action(anonFileAction);

src/cli/dbzar.ts

@@ -3,11 +3,13 @@
 import process from 'node:process';
 import {Command} from 'commander';
 import {anonColCommand} from './commands/anon-col/anon-col';
-import {anonDbCommand} from './commands/anon-db/anon-db-command';
+import {anonDbCommand} from './commands/anon-db/anon-db.command';
+import {anonFileCommand} from './commands/anon-file/anon-file.command';
 
 const program = new Command();
 
 program.addCommand(anonColCommand);
 program.addCommand(anonDbCommand);
+program.addCommand(anonFileCommand);
 
 program.parse(process.argv);

src/config/types.ts

@@ -1,6 +1,6 @@
 import {type Provider, type ProviderType} from '../types/types';
 
-export enum EngineType {
+export enum DatabaseEngineType {
 	PostGres = 'postgres',
 	Mongo = 'mongo',
 	MariaDB = 'mariadb',

src/processors/mariadb/mariadb-processor.ts → src/processors/databases/mariadb/mariadb-processor.ts

@@ -1,8 +1,8 @@
 import knex, {type Knex} from 'knex';
-import {type Anonymizer} from '../../anonymizers/types';
-import {debugLogger} from '../../services/loggers/debug-logger';
-import {BaseProcessor} from '../base-processor/base-processor';
-import {type Processor} from '../base-processor/processor';
+import {type Anonymizer} from '../../../anonymizers/types';
+import {debugLogger} from '../../../services/loggers/debug-logger';
+import {BaseProcessor} from '../../base-processor/base-processor';
+import {type Processor} from '../../base-processor/processor';
 
 const logger = debugLogger.extend('mariadb-processor');
 

src/processors/mongo/__tests__/mongo-processor.spec.ts → src/processors/databases/mongo/__tests__/mongo-processor.spec.ts

@@ -1,7 +1,7 @@
 import process from 'process';
-import {type Db, MongoClient, type MongoClientOptions} from 'mongodb';
-import {MaskAnonymizer} from '../../../anonymizers/mask/mask-anonymizer';
-import {type Anonymizer} from '../../../anonymizers/types';
+import {MongoClient, type Db, type MongoClientOptions} from 'mongodb';
+import {MaskAnonymizer} from '../../../../anonymizers/mask/mask-anonymizer';
+import {type Anonymizer} from '../../../../anonymizers/types';
 import {MongoProcessor} from '../mongo-processor';
 
 describe('mongo-processor', () => {

src/processors/mongo/mongo-processor.ts → src/processors/databases/mongo/mongo-processor.ts

@@ -1,8 +1,8 @@
-import {type Db, MongoClient} from 'mongodb';
-import {type Anonymizer} from '../../anonymizers/types';
-import {debugLogger} from '../../services/loggers/debug-logger';
-import {BaseProcessor} from '../base-processor/base-processor';
-import {type Processor} from '../base-processor/processor';
+import {MongoClient, type Db} from 'mongodb';
+import {type Anonymizer} from '../../../anonymizers/types';
+import {debugLogger} from '../../../services/loggers/debug-logger';
+import {BaseProcessor} from '../../base-processor/base-processor';
+import {type Processor} from '../../base-processor/processor';
 
 const logger = debugLogger.extend('mongo-processor');
 export class MongoProcessor extends BaseProcessor implements Processor {

src/processors/databases/postgres/__tests__/postgres-processor.spec.ts

@@ -0,0 +1,88 @@
+import {type Knex} from 'knex';
+import {newDb} from 'pg-mem';
+import {MaskAnonymizer} from '../../../../anonymizers/mask/mask-anonymizer';
+import {type Anonymizer} from '../../../../anonymizers/types';
+import {PostgresProcessor} from '../postgres/postgres-processor';
+
+describe('PostgresProcessor', () => {
+	let knex: Knex;
+
+	beforeEach(async () => {
+		// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
+		knex = newDb().adapters.createKnex();
+	});
+
+	afterEach(async () => {
+		await knex.destroy();
+	});
+
+	it('should process a single doc', async () => {
+		await knex.schema.createTable('users', (table) => {
+			table.increments('id');
+			table.string('firstName');
+		});
+
+		await knex('users').insert({firstName: 'John'});
+
+		const selectedRows1 = await knex('users').select('firstName');
+		expect(selectedRows1[0].firstName).toBe('John');
+
+		const spy1 = jest
+			.spyOn(knex.client, 'destroy')
+			.mockImplementationOnce(async () => {
+				// Do nothing
+			});
+		const spy2 = jest
+			.spyOn(PostgresProcessor.prototype as any, 'buildClient')
+			.mockImplementationOnce(() => knex);
+
+		// Anonymize the db
+		const processor: PostgresProcessor = new PostgresProcessor(
+			'postgresql://localhost',
+		);
+
+		const anonymizer: Anonymizer = new MaskAnonymizer();
+		await processor.processColumn('users', 'firstName', anonymizer);
+
+		spy1.mockRestore();
+		spy2.mockRestore();
+
+		// Check again
+		const selectedRows2 = await knex('users').select('firstName');
+		expect(selectedRows2[0].firstName).toBe('****');
+	});
+
+	it('should process multiple rows', async () => {
+		await knex.schema.createTable('users', (table) => {
+			table.increments('id');
+			table.string('firstName');
+		});
+
+		await knex('users').insert([{firstName: 'test1'}, {firstName: 'test2'}]);
+
+		const spy1 = jest
+			.spyOn(knex.client, 'destroy')
+			.mockImplementationOnce(async () => {
+				// Do nothing
+			});
+		const spy2 = jest
+			.spyOn(PostgresProcessor.prototype as any, 'buildClient')
+			.mockImplementationOnce(() => knex);
+
+		// Anonymize the db
+		const processor: PostgresProcessor = new PostgresProcessor(
+			'postgresql://localhost',
+		);
+
+		const anonymizer: Anonymizer = new MaskAnonymizer();
+		await processor.processColumn('users', 'firstName', anonymizer);
+
+		spy1.mockRestore();
+		spy2.mockRestore();
+
+		// Check again
+		const selectedRows2 = await knex('users').select('firstName');
+		expect(selectedRows2[0].firstName).toBe('*****');
+		expect(selectedRows2[1].firstName).toBe('*****');
+	});
+});

src/processors/postgres/__tests__/postgres-processor.spec.ts → src/processors/databases/postgres/postgres/__tests__/postgres-processor.spec.ts

@@ -1,7 +1,8 @@
 import {type Knex} from 'knex';
 import {newDb} from 'pg-mem';
-import {MaskAnonymizer} from '../../../anonymizers/mask/mask-anonymizer';
-import {type Anonymizer} from '../../../anonymizers/types';
+
+import {MaskAnonymizer} from '../../../../../anonymizers/mask/mask-anonymizer';
+import {type Anonymizer} from '../../../../../anonymizers/types';
 import {PostgresProcessor} from '../postgres-processor';
 
 describe('PostgresProcessor', () => {

src/processors/postgres/postgres-processor.ts → src/processors/databases/postgres/postgres/postgres-processor.ts

@@ -1,8 +1,8 @@
 import knex, {type Knex} from 'knex';
-import {type Anonymizer} from '../../anonymizers/types';
-import {debugLogger} from '../../services/loggers/debug-logger';
-import {BaseProcessor} from '../base-processor/base-processor';
-import {type Processor} from '../base-processor/processor';
+import {type Anonymizer} from '../../../../anonymizers/types';
+import {debugLogger} from '../../../../services/loggers/debug-logger';
+import {BaseProcessor} from '../../../base-processor/base-processor';
+import {type Processor} from '../../../base-processor/processor';
 
 const logger = debugLogger.extend('postgress-processor');
 

src/processors/types/collection.ts → src/processors/databases/types/collection.ts

@@ -1,4 +1,4 @@
-import {type Anonymizer} from '../../anonymizers/types';
+import {type Anonymizer} from '../../../anonymizers/types';
 
 export type Collection = {
 	dbName: string;

src/processors/utils/get-collections.ts → src/processors/databases/utils/get-collections.ts

@@ -1,6 +1,6 @@
-import {type Anonymizer} from '../../anonymizers/types';
-import {createAnonymizer} from '../../anonymizers/utils/create-anonymizer';
-import {type Config} from '../../config/types';
+import {type Anonymizer} from '../../../anonymizers/types';
+import {createAnonymizer} from '../../../anonymizers/utils/create-anonymizer';
+import {type Config} from '../../../config/types';
 import {type Collection} from '../types/collection';
 
 export function getCollections(config: Config): Collection[] {