97 add engine command (#101)

* clean cli

* commands

* file options

* add

* command

* dev

* non working

* remove

* sqlite is now file

* add

* rename

* rename

* move

* set

* better commands

* move anon-col up

* fix

* remove duplicat

* inside

* set

* set

* remove engine from connection options

* remove column from connection options as well

* knex

* change

* set

* change

* simple

* fix

* shift

* rename

* set

* set

* change connection options

* dep

* broken

* broken

* build

* fix

* dumber

* show options

* name

* set

* revieve mongo

* set

* base

* shorter

* fix knex

* test

* set

* scramble

* remove this

* set

* fix

* internal

* remove

* fix

* test

* option

* required

* back

* remove

* path

* set

* restore

* build sqlite with connection string

* remove unused

* flat

* set

* add sourcemaps

* ignore launch

* fix

* fix

* simple to col

* mutable

* restore

* remove

* remove

* set

* better engine

* remove shortcut

* add scramble

* move sqlite as db

* use table

* short

* implict

* file

* keep files for sqlite

* revert

* exp

* change

* add to db

* same options

* extract

* add mask

* set

* set

* bind to workspace

* set

* set

* change

.changeset/stupid-books-punch.md

@@ -0,0 +1,5 @@
+---
+"@databye/cli": major
+---
+
+changed cli to new format

.changeset/tall-schools-notice.md

@@ -0,0 +1,5 @@
+---
+"@databye/sqlite": patch
+---
+
+set new connections

.vscode/.gitignore

@@ -0,0 +1 @@
+launch.json

README.md

@@ -59,7 +59,12 @@ npx databye anon-col mask -u postgresql:/localhost -db test -t users -col firstN
 
 Scramble `lastName` column in `users` table in `test` db:
 ```
-npx databye anon-col scramble -u mongodb://localhost -db test -t users -col lastName
+npx databye anon-col mongo 
+  --uri mongodb://localhost
+  --database test 
+  --table users 
+  --column lastName
+  scramble
 
 // { "lastName": "Smith" } => { "lastName": "hSmti" }
 ```
@@ -68,14 +73,21 @@ npx databye anon-col scramble -u mongodb://localhost -db test -t users -col last
 
 Mask `firstName` column in `users` table in `dev.db`:
 ```
-npx databye anon-col mask -e sqlite -f /home/dev.db -t users -col firstName
+npx databye anon-col sqlite
+   --uri /home/dev.db
+   --table users
+   --column firstName
+    mask
 ```
 
 ## CSV
 
 Mask `email` column in `file.csv`:
 ```
-npx databye anon-col mask  -e csv -f /home/file.csv -col email
+npx databye anon-col csv 
+  --file /home/file.csv 
+  --column email 
+  mask
 ```
 
 # Anonymizers

packages/cli/package.json

@@ -21,6 +21,8 @@
   "scripts": {
     "clean": "rimraf dist",
     "build": "tsc",
+    "dev": "tsx -r dotenv/config src/dev-cli.ts",
+    "dev:cli": "node ./dist/dev-cli.js",
     "build:deps": "pnpm -F cli... build",
     "build:deploy": "pnpm deploy -F cli ./out",
     "build:watch": "tsc --watch"
@@ -34,7 +36,7 @@
     "@databye/mssql": "workspace:^",
     "@databye/postgres": "workspace:^",
     "@databye/processor": "workspace:^",
-    "@databye/sqlite": "^0.0.1",
+    "@databye/sqlite": "workspace:^",
     "chalk": "^5.3.0",
     "commander": "^12.1.0",
     "connection-string": "^4.4.0",

packages/cli/src/anon-col/anon-col-command.ts

@@ -0,0 +1,16 @@
+import { Command } from "commander";
+import { addEngineCommands } from "./engine/add-engine-commands.js";
+
+export const anonColCommand: Command = new Command("anon-col");
+
+anonColCommand
+  .description("Anonymize a single column in a table")
+  .option("--confirm", "Confirm before running", true)
+  .option("--no-confirm", "skip confirmation");
+
+const engineCommands = addEngineCommands(anonColCommand);
+
+// add commands to parent
+for (const engineCommand of engineCommands) {
+  anonColCommand.addCommand(engineCommand);
+}

packages/cli/src/anon-col/column-processor-runner.ts

@@ -0,0 +1,22 @@
+import { type Anonymizer } from "@databye/anonymizers";
+import { BaseColumnProcessor } from "@databye/processor";
+import ora from "ora";
+import { checkUserConfirm } from "./helpers/check-user-confirm.js";
+
+export class ColumnProcessorRunner {
+  constructor(
+    private readonly columnProcessor: BaseColumnProcessor,
+    private readonly anonymizer: Anonymizer
+  ) {}
+
+  async processColumn(columnName: string, checkConfirm = true) {
+    if (checkConfirm) {
+      const isConfirmed: boolean = await checkUserConfirm(columnName);
+      if (!isConfirmed) return;
+    }
+
+    const spinner = ora("Anonymizing column").start();
+    await this.columnProcessor.anonymizeColumn(columnName, this.anonymizer);
+    spinner.stop();
+  }
+}

packages/cli/src/anon-col/engine/add-engine-commands.ts

@@ -0,0 +1,45 @@
+import { EngineType } from "@databye/common";
+import { Command } from "commander";
+import { addAnonymizerCommands } from "../../anoymizers/add-anonymizer-commands.js";
+import { addColumnOption } from "../helpers/add-column-option.js";
+import { addConnectionOptions } from "../helpers/add-connection-options.js";
+import { addFileOptions } from "../helpers/add-file-options.js";
+
+const databaseEngines: EngineType[] = [
+  EngineType.PostGres,
+  EngineType.Mongo,
+  EngineType.MariaDB,
+  EngineType.MSSQL,
+  EngineType.MySQL,
+  EngineType.SQLite,
+];
+
+const fileEngines: EngineType[] = [EngineType.CSV];
+
+function createDatabaseCommands(): Command[] {
+  const commands = Object.values(databaseEngines).map((val) => {
+    let command = new Command(val);
+    addConnectionOptions(command);
+    addColumnOption(command);
+    return command;
+  });
+  return commands;
+}
+
+function createFileCommands(): Command[] {
+  const commands = Object.values(fileEngines).map((val) => {
+    let command = new Command(val);
+    addColumnOption(command);
+    addFileOptions(command);
+    return command;
+  });
+  return commands;
+}
+
+export function addEngineCommands(cmd: Command) {
+  let engineCommands = [...createDatabaseCommands(), ...createFileCommands()];
+
+  engineCommands.forEach((cmd) => addAnonymizerCommands(cmd));
+
+  return engineCommands;
+}

packages/cli/src/anon-col/helpers/add-column-option.ts

@@ -0,0 +1,5 @@
+import { Command } from "commander";
+
+export function addColumnOption(cmd: Command) {
+  cmd.requiredOption("-col --column <columnName>", "Column name to process");
+}

packages/cli/src/anon-col/helpers/add-connection-options.ts

@@ -0,0 +1,11 @@
+import { type Command } from "commander";
+
+export function addConnectionOptions(cmd: Command): Command {
+  return cmd
+    .option("--database <databaseName>", "Database name")
+    .option("--password <password>", "database password")
+    .option("--server <serverName>", "server to connect to")
+    .option("--table <tableName>", "Table name")
+    .option("--uri <connectionString>", "Connection string")
+    .option("--user <userName>", "Username to use");
+}

packages/cli/src/anon-col/helpers/add-file-options.ts

@@ -0,0 +1,5 @@
+import { type Command } from "commander";
+
+export function addFileOptions(cmd: Command) {
+  cmd.requiredOption("--file <filePath>", "File path");
+}

packages/cli/src/anon-col/helpers/check-user-confirm.ts

@@ -0,0 +1,30 @@
+import chalk from "chalk";
+import Enquirer from "enquirer";
+
+const { prompt } = Enquirer;
+
+// Cyan
+const color = chalk.cyan;
+
+export async function checkUserConfirm(column: string): Promise<boolean> {
+  try {
+    let confirmMessage = `Are you sure you want to anonymize column "${color(
+      column
+    )}"`;
+
+    const answer = await prompt<{ run: boolean }>({
+      type: "confirm",
+      name: "run",
+      message: confirmMessage,
+      initial: "true",
+    });
+    if (answer.run) {
+      return true;
+    }
+
+    return false;
+  } catch (error: unknown) {
+    console.error(error);
+    throw new Error("could not receive answer");
+  }
+}

packages/cli/src/anon-col/helpers/create-processor.ts

@@ -0,0 +1,83 @@
+import { EngineType, FileOptions, createLogger } from "@databye/common";
+import { CSVProcessor } from "@databye/csv";
+import { MariaDatabaseProcessor } from "@databye/mariadb";
+import { MongoProcessor } from "@databye/mongo";
+import { MsSqlProcessor } from "@databye/mssql";
+import { PostgresProcessor } from "@databye/postgres";
+import { BaseColumnProcessor } from "@databye/processor";
+import { SQLiteProcessor } from "@databye/sqlite";
+import {
+  CliConnectionOptions,
+  extractConnectionOptions,
+} from "./extract-connection-options.js";
+
+const logger = createLogger();
+
+export function createProcessor(
+  engineType: EngineType,
+  options: unknown
+): BaseColumnProcessor {
+  logger.debug(`engine = ${engineType}`);
+  switch (engineType) {
+    case EngineType.Mongo: {
+      const connectionsOptions = extractConnectionOptions(
+        options as CliConnectionOptions
+      );
+      logger.debug("creating mongo processor");
+      if (!connectionsOptions.connectionString)
+        throw new Error("invalid connection string uri");
+      if (!connectionsOptions.databaseName) throw new Error("no db");
+      if (!connectionsOptions.tableName) throw new Error("no table");
+      return new MongoProcessor(
+        connectionsOptions.connectionString,
+        connectionsOptions.databaseName,
+        connectionsOptions.tableName
+      );
+    }
+
+    case EngineType.PostGres: {
+      logger.debug("creating postgres processor");
+      const connectionsOptions = extractConnectionOptions(
+        options as CliConnectionOptions
+      );
+
+      return new PostgresProcessor(connectionsOptions);
+    }
+
+    case EngineType.MariaDB:
+    case EngineType.MySQL: {
+      logger.debug("creating mariadb processor");
+      const connectionsOptions = extractConnectionOptions(
+        options as CliConnectionOptions
+      );
+      return new MariaDatabaseProcessor(connectionsOptions);
+    }
+
+    case EngineType.MSSQL: {
+      logger.debug("creating mssql processor");
+      const connectionsOptions = extractConnectionOptions(
+        options as CliConnectionOptions
+      );
+      return new MsSqlProcessor(connectionsOptions);
+    }
+
+    case EngineType.SQLite: {
+      logger.debug("creating sqlite processor");
+      logger.debug(`options = ${JSON.stringify(options, null, 2)}`);
+      const connectionsOptions = extractConnectionOptions(
+        options as CliConnectionOptions
+      );
+      logger.debug(
+        `connectionsOptions = ${JSON.stringify(connectionsOptions, null, 2)}`
+      );
+      return new SQLiteProcessor(connectionsOptions);
+    }
+
+    case EngineType.CSV: {
+      logger.debug("creating csv processor");
+      logger.debug(`options = ${JSON.stringify(options, null, 2)}`);
+      const fileOptions = options as FileOptions;
+      return new CSVProcessor(fileOptions.file);
+    }
+  }
+}

packages/cli/src/anon-col/helpers/extract-connection-options.ts

@@ -0,0 +1,30 @@
+import { createLogger, type ConnectionOptions } from "@databye/common";
+
+export type CliConnectionOptions = {
+  database: string;
+  password?: string;
+  server?: string;
+  table: string;
+  uri?: string;
+  user?: string;
+};
+
+const logger = createLogger();
+
+export function extractConnectionOptions(
+  cliOptions: CliConnectionOptions
+): ConnectionOptions {
+  logger.debug(
+    `cli connection options = ${JSON.stringify(cliOptions, null, 2)}`
+  );
+  const { uri, database, password, server, table, user } = cliOptions;
+
+  return {
+    databaseName: database,
+    password,
+    serverName: server,
+    tableName: table,
+    connectionString: uri,
+    userName: user,
+  };
+}

packages/cli/src/anoymizers/add-anonymizer-commands.ts

@@ -0,0 +1,10 @@
+import { Command } from "commander";
+import { getMaskCommand } from "./mask/mask-command.js";
+import { getScrambleCommand } from "./scramble/scramble-command.js";
+
+export function addAnonymizerCommands(cmd: Command) {
+  const scrambleCommand = getScrambleCommand();
+  const maskCommand = getMaskCommand();
+  cmd.addCommand(scrambleCommand);
+  cmd.addCommand(maskCommand);
+}

packages/cli/src/anoymizers/mask/mask-action.ts

@@ -0,0 +1,20 @@
+import {
+  createMaskAnonymizer,
+  MaskAnonymizer,
+  type MaskOptions,
+} from "@databye/anonymizers";
+import { createLogger } from "@databye/common";
+import { type Command } from "commander";
+import { runAnonymizerCommand } from "../run-anonymizer-command.js";
+
+const logger = createLogger();
+
+export async function maskAction(this: Command) {
+  const maskOptions: MaskOptions = {
+    character: this.opts().character as string,
+  };
+
+  // // Build anonymizer
+  const maskAnonymizer: MaskAnonymizer = createMaskAnonymizer(maskOptions);
+  await runAnonymizerCommand(this.parent!, maskAnonymizer);
+}

packages/cli/src/anoymizers/mask/mask-command.ts

@@ -0,0 +1,14 @@
+import { defaultMaskOptions } from "@databye/anonymizers";
+import { Command } from "commander";
+import { maskAction } from "./mask-action.js";
+
+export function getMaskCommand(): Command {
+  return new Command("mask")
+    .description("mask a single column")
+    .option(
+      "-c --character <char>",
+      "Character to mask with",
+      defaultMaskOptions.character
+    )
+    .action(maskAction);
+}

packages/cli/src/anoymizers/run-anonymizer-command.ts

@@ -0,0 +1,22 @@
+import { Anonymizer } from "@databye/anonymizers";
+import { createLogger, EngineType } from "@databye/common";
+import { Command } from "commander";
+import { ColumnProcessorRunner } from "../anon-col/column-processor-runner.js";
+import { createProcessor } from "../anon-col/helpers/create-processor.js";
+
+const logger = createLogger();
+
+export async function runAnonymizerCommand(
+  engineCommand: Command,
+  anonymizer: Anonymizer
+) {
+  const engineType = engineCommand.name() as EngineType;
+  logger.debug(`engineType = ${engineType}`);
+  const engineOptions = engineCommand?.optsWithGlobals();
+  const columnProcessor = createProcessor(engineType, engineOptions);
+  const runner = new ColumnProcessorRunner(columnProcessor, anonymizer);
+  const isConfirmed = engineCommand.optsWithGlobals().confirm as boolean;
+  const columnName = engineCommand.optsWithGlobals().column as string;
+  logger.debug(`columnName=${columnName} isConfirmed=${isConfirmed}`);
+  await runner.processColumn(columnName, isConfirmed);
+}