Feat(analysis): Add --large merge - merge multiple project files in a divided cc.json (#3743)

README.md

@@ -5,7 +5,7 @@
 
   <p>
     Latest Release: <br>
-    Analysis <a href="https://github.com/MaibornWolff/codecharta/releases/tag/ana-1.129.0">1.129.0</a> | Visualization <a href="https://github.com/MaibornWolff/codecharta/releases/tag/vis-1.131.2">1.131.2</a> 
+    Analysis <a href="https://github.com/MaibornWolff/codecharta/releases/tag/ana-1.129.0">1.129.0</a> | Visualization <a href="https://github.com/MaibornWolff/codecharta/releases/tag/vis-1.131.2">1.131.2</a>
 
 [comment]: ##################################################################################
 [comment]: <Ensure that the words 'latest release' are above the line with the links>

analysis/CHANGELOG.md

@@ -7,8 +7,17 @@ and this project adheres to [Semantic Versioning](http://semver.org/)
 
 ## [unreleased] (Added 🚀 | Changed | Removed 🗑 | Fixed 🐞 | Chore 👨‍💻 👩‍💻)
 
+### Added 🚀
+
+- Add a new `--large` flat to the MergeFilter that merges projects into one file each in its own subfolder depending on the input file's dot-prefix name [#3841](https://github.com/MaibornWolff/codecharta/pull/3841)
+- Add the ability to the MergeFilter to specify the output file during `--mimo` operation [#3841](https://github.com/MaibornWolff/codecharta/pull/3841)
+
 ## [1.129.0] - 2024-11-29
 
+### Added 🚀
+
+- Add a new `--mimo` flag to the MergeFilter that merges multiple project files into multiple output files depending on the input file's dot-prefix name [#3800](https://github.com/MaibornWolff/codecharta/pull/3800)
+
 ### Changed
 
 - Moved the structure print functionality to a new tool called 'inspection' [#3826](https://github.com/MaibornWolff/codecharta/pull/3826)

analysis/filter/MergeFilter/README.md

@@ -14,19 +14,20 @@ Both strategies will merge the unique list entries for `attributeTypes` and `bla
 
 ## Usage and Parameters
 
-| Parameters                      | Description                                                          |
-|---------------------------------|----------------------------------------------------------------------|
-| `FILE`                          | files to merge                                                       |
-| `-a, --add-missing`             | [Leaf Merging Strategy] enable adding missing nodes to reference     |
-| `-h, --help`                    | displays help and exits                                              |
-| `--ignore-case`                 | ignores case when checking node names                                |
-| `--leaf`                        | use leaf merging strategy                                            |
-| `-nc, --not-compressed`         | save uncompressed output File                                        |
-| `-o, --outputFile=<outputFile>` | output File (or empty for stdout; ignored in [MIMO mode]))           |
-| `--recursive`                   | use recursive merging strategy (default)                             |
-| `--mimo`                        | merge multiple files with the same prefix into multiple output files |
-| `-ld, --levenshtein-distance`   | [MIMO mode] levenshtein distance for name match suggestions          |
-| `-f`                            | force merge non-overlapping modules at the top-level structure       |
+| Parameters                      | Description                                                                      |
+|---------------------------------|----------------------------------------------------------------------------------|
+| `FILE`                          | files to merge                                                                   |
+| `-a, --add-missing`             | [Leaf Merging Strategy] enable adding missing nodes to reference                 |
+| `-h, --help`                    | displays help and exits                                                          |
+| `--ignore-case`                 | ignores case when checking node names                                            |
+| `--leaf`                        | use leaf merging strategy                                                        |
+| `-nc, --not-compressed`         | save uncompressed output File                                                    |
+| `-o, --outputFile=<outputFile>` | output File (or empty for stdout; [MIMO mode] output folder))                    |
+| `--recursive`                   | use recursive merging strategy (default)                                         |
+| `--mimo`                        | merge multiple files with the same prefix into multiple output files             |
+| `-ld, --levenshtein-distance`   | [MIMO mode] levenshtein distance for name match suggestions                      |
+| `-f`                            | force merge non-overlapping modules at the top-level structure                   |
+| `--large`                       | merge multiple project files into one output file, separated by their dot-prefix |
 
 ```
 Usage: ccsh merge [-ah] [--ignore-case] [--leaf] [-nc] [--recursive]
@@ -53,10 +54,24 @@ This last example inputs the folder foo, which will result in all project files
 ccsh merge myProjectFolder/ --mimo -ld 0 -f
 ```
 
-## MIMO - Multiply Inputs Multiple Outputs
+## MIMO Merge - Multiply Inputs Multiple Outputs
 
 Matches multiple `cc.json` files based on their prefix (e.g. **myProject**.git.cc.json). Tries to match project names with typos and asks which to add to the output.
 If you want to use this in a CI/CD pipeline environment you may find it useful to specify `-ld` and `-f` to not prompt any user input.
 The output file name follows the following schema: `myProject.merge.cc.json`.
 
-> IMPORTANT: Output is always the current working directory.
+## Large Merge
+
+Merges multiple `.cc.json` files into one projects, but separates them into sub-folders, with names defined through the dot-prefixes of the input files:
+
+```
+ccsh merge aa.cc.json bb.cc.json cc.cc.json --large -o myOutputFile -nc
+# myOutputFile.cc.json:
+# - root
+# - - aa
+# - - - *
+# - - bb
+# - - - *
+# - - cc
+# - - - *
+```

analysis/filter/MergeFilter/src/main/kotlin/de/maibornwolff/codecharta/filter/mergefilter/LargeMerge.kt

@@ -0,0 +1,54 @@
+package de.maibornwolff.codecharta.filter.mergefilter
+
+import de.maibornwolff.codecharta.model.BlacklistItem
+import de.maibornwolff.codecharta.model.Edge
+import de.maibornwolff.codecharta.model.Node
+import de.maibornwolff.codecharta.model.NodeType
+import de.maibornwolff.codecharta.model.Project
+
+class LargeMerge {
+    companion object {
+        fun packageProjectInto(project: Project, prefix: String): Project {
+            val modifiedProject = Project(
+                project.projectName,
+                moveNodesIntoFolder(project.rootNode, prefix),
+                project.apiVersion,
+                addFolderToEdgePaths(project.edges, prefix),
+                project.attributeTypes,
+                project.attributeDescriptors,
+                addFolderToBlackListPaths(project.blacklist, prefix)
+            )
+            return modifiedProject
+        }
+
+        private fun moveNodesIntoFolder(root: Node, folderName: String): List<Node> {
+            val newRoot = Node(
+                name = folderName,
+                type = NodeType.Folder,
+                children = root.children
+            )
+            val mutableRoot = root.toMutableNode()
+            mutableRoot.children = mutableSetOf(newRoot.toMutableNode())
+            return listOf(mutableRoot.toNode())
+        }
+
+        private fun addFolderToEdgePaths(edges: List<Edge>, folderName: String): List<Edge> {
+            edges.forEach {
+                it.fromNodeName = insertFolderIntoPath(it.fromNodeName, folderName)
+                it.toNodeName = insertFolderIntoPath(it.toNodeName, folderName)
+            }
+            return edges
+        }
+
+        private fun addFolderToBlackListPaths(blacklist: List<BlacklistItem>, folderName: String): List<BlacklistItem> {
+            blacklist.forEach {
+                it.path = insertFolderIntoPath(it.path, folderName)
+            }
+            return blacklist
+        }
+
+        private fun insertFolderIntoPath(path: String, folderName: String): String {
+            return path.replace(Regex("^/root/"), "/root/$folderName/")
+        }
+    }
+}

analysis/filter/MergeFilter/src/main/kotlin/de/maibornwolff/codecharta/filter/mergefilter/MergeFilter.kt

@@ -58,6 +58,12 @@ class MergeFilter(
     )
     private var levenshteinDistance = 3
 
+    @CommandLine.Option(
+        names = ["--large"],
+        description = ["Merges input files into on file divided into project sub-folder as defined per prefix of each input file"]
+    )
+    private var largeMerge = false
+
     override val name = NAME
     override val description = DESCRIPTION
 
@@ -94,8 +100,11 @@ class MergeFilter(
 
         if (mimo) {
             processMimoMerge(sourceFiles, nodeMergerStrategy)
+        } else if (largeMerge) {
+            processLargeMerge(sourceFiles, nodeMergerStrategy)
         } else {
             val projects = readInputFiles(sourceFiles)
+
             if (!continueIfIncompatibleProjects(projects)) return null
 
             val mergedProject = ProjectMerger(projects, nodeMergerStrategy).merge()
@@ -155,24 +164,48 @@ class MergeFilter(
                 return@forEach
             }
 
-            val projects = readInputFiles(confirmedFileList)
-            if (projects.size <= 1) {
+            val projectsFileNamePairs = readInputFilesKeepFileNames(confirmedFileList)
+            val projects = projectsFileNamePairs.map { it.second }
+            if (projectsFileNamePairs.size <= 1) {
                 Logger.warn { "After deserializing there were one or less projects. Continue with next group" }
                 return@forEach
             }
 
             if (!continueIfIncompatibleProjects(projects)) return@forEach
 
             val mergedProject = ProjectMerger(projects, nodeMergerStrategy).merge()
-            val outputFilePrefix = Mimo.retrieveGroupName(confirmedFileList)
-            ProjectSerializer.serializeToFileOrStream(mergedProject, "$outputFilePrefix.merge.cc.json", output, compress)
+            val outputFilePrefix = Mimo.retrieveGroupName(projectsFileNamePairs.map { it.first })
+            val outputFileName = "$outputFilePrefix.merge.cc.json"
+            val outputFilePath = Mimo.assembleOutputFilePath(outputFile, outputFileName)
+            ProjectSerializer.serializeToFileOrStream(mergedProject, outputFilePath, output, compress)
             Logger.info {
                 "Merged files with prefix '$outputFilePrefix' into" +
-                    " '$outputFilePrefix.merge.cc.json${if (compress) ".gz" else ""}'"
+                    " '$outputFileName${if (compress) ".gz" else ""}'"
             }
         }
     }
 
+    private fun processLargeMerge(sourceFiles: List<File>, nodeMergerStrategy: NodeMergerStrategy) {
+        val projectsFileNamePairs = readInputFilesKeepFileNames(sourceFiles)
+        val fileNameList = projectsFileNamePairs.map { it.first }
+
+        require(fileNameList.size > 1) {
+            Logger.warn { "One or less projects in input, merging aborted." }
+        }
+
+        require(fileNameList.groupingBy { it.substringBefore(".") }.eachCount().all { it.value == 1 }) {
+            Logger.warn { "Make sure that the input prefixes across all input files are unique!" }
+        }
+
+        val packagedProjects: MutableList<Project> = mutableListOf()
+        projectsFileNamePairs.forEach {
+            packagedProjects.add(LargeMerge.packageProjectInto(it.second, it.first.substringBefore(".")))
+        }
+
+        val mergedProject = ProjectMerger(packagedProjects, nodeMergerStrategy).merge()
+        ProjectSerializer.serializeToFileOrStream(mergedProject, outputFile, output, compress)
+    }
+
     private fun readInputFiles(files: List<File>): List<Project> {
         return files.mapNotNull {
             val input = it.inputStream()
@@ -184,4 +217,15 @@ class MergeFilter(
             }
         }
     }
+
+    private fun readInputFilesKeepFileNames(files: List<File>): List<Pair<String, Project>> {
+        return files.mapNotNull {
+            try {
+                Pair(it.name, ProjectDeserializer.deserializeProject(it.inputStream()))
+            } catch (e: Exception) {
+                Logger.warn { "${it.name} is not a valid project file and will be skipped." }
+                null
+            }
+        }
+    }
 }

analysis/filter/MergeFilter/src/main/kotlin/de/maibornwolff/codecharta/filter/mergefilter/ParserDialog.kt

@@ -20,24 +20,32 @@ class ParserDialog {
                 inputFolderName = getInputFileName("cc.json", InputType.FOLDER)
             } while (!InputHelper.isInputValidAndNotNull(arrayOf(File(inputFolderName)), canInputContainFolders = true))
 
-            val isMimoMode = KInquirer.promptConfirm(
-                message = "Do you want to use MIMO mode? (multiple inputs multiple outputs)",
-                default = false
+            val defaultMerge = "Default merging..."
+            val mimoMerge = "Mimo Merge"
+            val largeMerge = "Large Merge"
+            val mergeMode = KInquirer.promptList(
+                message = "Do you want to use a special merge mode?",
+                choices = listOf(defaultMerge, mimoMerge, largeMerge)
             )
 
-            var outputFileName = ""
+            val outputFileName: String
             val isCompressed: Boolean
             var levenshteinDistance = 0
-            if (isMimoMode) {
-                levenshteinDistance = KInquirer.promptInputNumber(
-                    message = "Select Levenshtein Distance for name match suggestions (0 for no suggestions)",
-                    default = "3"
-                ).toInt()
+            if (mergeMode == mimoMerge) {
+                outputFileName = KInquirer.promptInput(
+                    message = "What is the output folder path?",
+                    hint = "Uses the current working directory if empty"
+                )
 
                 isCompressed = KInquirer.promptConfirm(
                     message = "Do you want to compress the output file(s)?",
                     default = true
                 )
+
+                levenshteinDistance = KInquirer.promptInputNumber(
+                    message = "Select Levenshtein Distance for name match suggestions (0 for no suggestions)",
+                    default = "3"
+                ).toInt()
             } else {
                 outputFileName =
                     KInquirer.promptInput(
@@ -82,18 +90,24 @@ class ParserDialog {
                 "--recursive=${!leafFlag}",
                 "--leaf=$leafFlag",
                 "--ignore-case=$ignoreCase",
-                "--not-compressed=$isCompressed"
+                "--not-compressed=$isCompressed",
+                "--output-file=$outputFileName"
             )
 
-            if (isMimoMode) {
+            if (mergeMode == mimoMerge) {
                 return basicMergeConfig + listOf(
                     "--mimo=true",
                     "--levenshtein-distance=$levenshteinDistance"
                 )
             }
-            return basicMergeConfig + listOf(
-                "--output-file=$outputFileName"
-            )
+
+            if (mergeMode == largeMerge) {
+                return basicMergeConfig + listOf(
+                    "--large=true"
+                )
+            }
+
+            return basicMergeConfig
         }
 
         fun askForceMerge(): Boolean {

analysis/filter/MergeFilter/src/main/kotlin/de/maibornwolff/codecharta/filter/mergefilter/mimo/Mimo.kt

@@ -51,8 +51,8 @@ class Mimo {
             }
         }
 
-        fun retrieveGroupName(files: List<File>): String {
-            val filePrefixes = files.map { it.name.substringBefore(".") }.toSet()
+        fun retrieveGroupName(files: List<String>): String {
+            val filePrefixes = files.map { it.substringBefore(".") }.toSet()
             if (filePrefixes.size == 1) return filePrefixes.first()
             return ParserDialog.askForMimoPrefix(filePrefixes)
         }
@@ -83,5 +83,15 @@ class Mimo {
 
             return cost[rhsLength]
         }
+
+        fun assembleOutputFilePath(filePath: String?, fileName: String): String {
+            return if (filePath.isNullOrEmpty()) {
+                fileName
+            } else if (File(filePath).isDirectory) {
+                "${File(filePath).path}/$fileName"
+            } else {
+                throw IllegalArgumentException("Please specify a folder for MIMO output or nothing")
+            }
+        }
     }
 }