Issue #33 - Add support for appendindex/prependindex

ppomes · Oct 10, 2024 · 3af7d05 · 3af7d05
1 parent a337e1e
commit 3af7d05
Show file tree

Hide file tree

Showing 11 changed files with 307 additions and 3 deletions.
diff --git a/Makefile.am b/Makefile.am
@@ -9,7 +9,7 @@ EXTRA_DIST = $(top_srcdir)/uthash/uthash.h \
              $(top_srcdir)/debian/copyright \
              $(top_srcdir)/debian/rules
 
-TULIST = test1 test1-insert-ignore null-example appendkey prependkey issue29 test_substring
+TULIST = test1 test1-insert-ignore null-example appendkey prependkey issue29 test_substring appendindex appendindex
 PYTHON_SUPPORT = @PYTHON_SUPPORT@
 JQ_SUPPORT = @JQ@
 

diff --git a/main/configparser.y b/main/configparser.y
@@ -91,7 +91,7 @@ static bool is_valid_json_path(const char *path);
 /* 
  * Flex tokens
  */
-%token SECRET STATS TABLES YES NO FIXEDNULL FIXED FIXEDQUOTED FIXEDUNQUOTED TEXTHASH EMAILHASH INTHASH TRUNCATE KEY APPENDKEY PREPENDKEY EQ LEFT RIGHT PYPATH PYSCRIPT PYDEF JSON PATH SEPARATEDBY SUBSTRING
+%token SECRET STATS TABLES YES NO FIXEDNULL FIXED FIXEDQUOTED FIXEDUNQUOTED TEXTHASH EMAILHASH INTHASH TRUNCATE KEY APPENDKEY PREPENDKEY APPENDINDEX PREPENDINDEX EQ LEFT RIGHT PYPATH PYSCRIPT PYDEF JSON PATH SEPARATEDBY SUBSTRING
 %token <strval> STRING IDENTIFIER
 %token <shortval> LENGTH
 
@@ -209,6 +209,8 @@ fieldaction:
   key |
   appendkey |
   prependkey |
+  appendindex |
+  prependindex |
   pydef |
   json |
 
@@ -280,6 +282,17 @@ prependkey:
                      workinfos.type = AM_PREPENDKEY;
                      STORE_FIXEDVALUE($2)
                     }
+appendindex:
+  APPENDINDEX STRING {
+                     workinfos.type = AM_APPENDINDEX;
+                     STORE_FIXEDVALUE($2)
+                   }
+prependindex:
+  PREPENDINDEX STRING {
+                     workinfos.type = AM_PREPENDINDEX;
+                     STORE_FIXEDVALUE($2)
+                    }
+
 pydef:
   PYDEF STRING {
                  #ifdef HAVE_PYTHON
@@ -370,4 +383,4 @@ static bool is_valid_json_path(const char *path) {
 
   return true;
 }
-#endif
+#endif
diff --git a/main/configscanner.l b/main/configscanner.l
@@ -64,6 +64,8 @@ truncate                { return TRUNCATE; }
 key                     { return KEY; }
 appendkey               { return APPENDKEY; }
 prependkey              { return PREPENDKEY; }
+appendindex             { return APPENDINDEX; }
+prependindex            { return PREPENDINDEX; }
 pydef                   { return PYDEF; }
 json                    { return JSON; }
 path                    { return PATH; }

diff --git a/main/dumpparser.y b/main/dumpparser.y
@@ -156,6 +156,7 @@ singlefield : VALUE {
       anonymized_res_st res_st;
       char *s;
       int nbcopied;
+      int rowindex;
       char concatvalue[ID_SIZE];
 #ifdef HAVE_JQ
       char *newjsonbackslash_str=NULL;
@@ -169,6 +170,7 @@ singlefield : VALUE {
 
       bool found=false;
       if (bfirstinsert) {
+        rowindex=0;
         for (cur=infos;cur!=NULL;cur=cur->hh.next) {
           if (memcmp(cur->key,currenttable,strlen(currenttable)) == 0) {
               if (cur->pos == currentfieldpos) {
@@ -187,6 +189,7 @@ singlefield : VALUE {
 
       if (found) {
         cur->infos.nbhits++;
+        rowindex++;
       }
 
       /* NULL values should remains NULL
@@ -298,6 +301,15 @@ singlefield : VALUE {
                  fprintf(stderr, "WARNING! Table %s fields order: for prependkey mode, the key must be defined before the field to anomymize\n",currenttable);
                }
                break;
+             case AM_APPENDINDEX:
+               nbcopied=snprintf(concatvalue,ID_SIZE,"%s%d",cur->infos.fixedvalue,rowindex);
+               quoted_output_helper(concatvalue,nbcopied,true);
+               break;
+             case AM_PREPENDINDEX:
+               nbcopied=snprintf(concatvalue,ID_SIZE,"%d%s",rowindex,cur->infos.fixedvalue);
+               quoted_output_helper(concatvalue,nbcopied,true);
+               break;
+
 #ifdef HAVE_JQ
              case AM_JSON:
                unquoted_json_str = mymalloc(curleng + 1);

diff --git a/main/myanon.h b/main/myanon.h
@@ -103,6 +103,8 @@ typedef enum anon_type
     AM_KEY,
     AM_APPENDKEY,
     AM_PREPENDKEY,
+    AM_APPENDINDEX,
+    AM_PREPENDINDEX,
     AM_SUBSTRING,
     AM_JSON,
 #ifdef HAVE_PYTHON

diff --git a/tests/appendindex.conf b/tests/appendindex.conf
@@ -0,0 +1,14 @@
+# Config file for concat.sql
+secret = 'lapin'
+stats  = 'no'
+
+tables = {
+   `players1` = {
+     `name` = appendindex 'player'     # concat string 'player' and the row index
+   }
+
+   `realnextgen` = {
+     `name` = appendindex 'nextgen'
+   }
+
+}
diff --git a/tests/appendindex.sql b/tests/appendindex.sql
@@ -0,0 +1,75 @@
+-- MySQL dump 10.13  Distrib 8.0.39, for Linux (x86_64)
+--
+-- Host: localhost    Database: appendindex
+-- ------------------------------------------------------
+-- Server version	8.0.39-0ubuntu0.24.04.2
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `players1`
+--
+
+DROP TABLE IF EXISTS `players1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `players1` (
+  `id` int NOT NULL,
+  `name` char(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `players1`
+--
+
+LOCK TABLES `players1` WRITE;
+/*!40000 ALTER TABLE `players1` DISABLE KEYS */;
+INSERT INTO `players1` VALUES (10,'Roger'),(20,'Rafael'),(30,'Novak'),(40,'Andy');
+/*!40000 ALTER TABLE `players1` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `realnextgen`
+--
+
+DROP TABLE IF EXISTS `realnextgen`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `realnextgen` (
+  `id` int NOT NULL,
+  `name` char(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `realnextgen`
+--
+
+LOCK TABLES `realnextgen` WRITE;
+/*!40000 ALTER TABLE `realnextgen` DISABLE KEYS */;
+INSERT INTO `realnextgen` VALUES (10,'Carlitos'),(20,'Jannick');
+/*!40000 ALTER TABLE `realnextgen` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2024-10-10 11:40:45
diff --git a/tests/appendindex_anon.sql b/tests/appendindex_anon.sql
@@ -0,0 +1,75 @@
+-- MySQL dump 10.13  Distrib 8.0.39, for Linux (x86_64)
+--
+-- Host: localhost    Database: appendindex
+-- ------------------------------------------------------
+-- Server version	8.0.39-0ubuntu0.24.04.2
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `players1`
+--
+
+DROP TABLE IF EXISTS `players1`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `players1` (
+  `id` int NOT NULL,
+  `name` char(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `players1`
+--
+
+LOCK TABLES `players1` WRITE;
+/*!40000 ALTER TABLE `players1` DISABLE KEYS */;
+INSERT INTO `players1` VALUES (10,'player1'),(20,'player2'),(30,'player3'),(40,'player4');
+/*!40000 ALTER TABLE `players1` ENABLE KEYS */;
+UNLOCK TABLES;
+
+--
+-- Table structure for table `realnextgen`
+--
+
+DROP TABLE IF EXISTS `realnextgen`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `realnextgen` (
+  `id` int NOT NULL,
+  `name` char(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `realnextgen`
+--
+
+LOCK TABLES `realnextgen` WRITE;
+/*!40000 ALTER TABLE `realnextgen` DISABLE KEYS */;
+INSERT INTO `realnextgen` VALUES (10,'nextgen1'),(20,'nextgen2');
+/*!40000 ALTER TABLE `realnextgen` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2024-10-10 11:40:45
diff --git a/tests/prependindex.conf b/tests/prependindex.conf
@@ -0,0 +1,9 @@
+# Config file for concat.sql
+secret = 'lapin'
+stats  = 'no'
+
+tables = {
+   `players` = {
+     `name` = prependindex 'player'     # concat the row index and string 'player'
+   }
+}
diff --git a/tests/prependindex.sql b/tests/prependindex.sql
@@ -0,0 +1,51 @@
+-- MySQL dump 10.13  Distrib 8.0.30, for Linux (x86_64)
+--
+-- Host: localhost    Database: concat
+-- ------------------------------------------------------
+-- Server version	8.0.30-0ubuntu0.22.04.1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `players`
+--
+
+DROP TABLE IF EXISTS `players`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `players` (
+  `id` int NOT NULL,
+  `name` char(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `players`
+--
+
+LOCK TABLES `players` WRITE;
+/*!40000 ALTER TABLE `players` DISABLE KEYS */;
+INSERT INTO `players` VALUES (10,'Roger'),(20,'Rafael'),(30,'Novak'),(40,'Andy');
+/*!40000 ALTER TABLE `players` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-08-04 21:32:09
diff --git a/tests/prependindex_anon.sql b/tests/prependindex_anon.sql
@@ -0,0 +1,51 @@
+-- MySQL dump 10.13  Distrib 8.0.30, for Linux (x86_64)
+--
+-- Host: localhost    Database: concat
+-- ------------------------------------------------------
+-- Server version	8.0.30-0ubuntu0.22.04.1
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!50503 SET NAMES utf8mb4 */;
+/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
+/*!40103 SET TIME_ZONE='+00:00' */;
+/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
+/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
+/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
+/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
+
+--
+-- Table structure for table `players`
+--
+
+DROP TABLE IF EXISTS `players`;
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!50503 SET character_set_client = utf8mb4 */;
+CREATE TABLE `players` (
+  `id` int NOT NULL,
+  `name` char(20) DEFAULT NULL,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+/*!40101 SET character_set_client = @saved_cs_client */;
+
+--
+-- Dumping data for table `players`
+--
+
+LOCK TABLES `players` WRITE;
+/*!40000 ALTER TABLE `players` DISABLE KEYS */;
+INSERT INTO `players` VALUES (10,'1player'),(20,'2player'),(30,'3player'),(40,'4player');
+/*!40000 ALTER TABLE `players` ENABLE KEYS */;
+UNLOCK TABLES;
+/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
+
+/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
+/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
+/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
+/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
+
+-- Dump completed on 2022-08-04 21:32:09