Automatic annotations and tagging (#1450)

* First cut for automatic annotations and tagging Signed-off-by: Prabhu Subramanian <[email protected]> * Prefer bom-link format for ml profile Signed-off-by: Prabhu Subramanian <[email protected]> * CBOM and ml-deep profile Signed-off-by: Prabhu Subramanian <[email protected]> --------- Signed-off-by: Prabhu Subramanian <[email protected]>
CycloneDX · Nov 12, 2024 · ab65c8b · ab65c8b
1 parent 21726cb
commit ab65c8b
Show file tree

Hide file tree

Showing 30 changed files with 862 additions and 57 deletions.
diff --git a/.github/workflows/repotests.yml b/.github/workflows/repotests.yml
@@ -288,7 +288,7 @@ jobs:
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-5.json --only spring
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-6.json --deep --evidence
           bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-7.json --profile research --export-proto
-          PREFER_MAVEN_DEPS_TREE=true bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-8.json --profile license-compliance
+          bin/cdxgen.js -p -t java repotests/java-sec-code -o bomresults/bom-java-sec-code-8.json --profile license-compliance
           bin/cdxgen.js -p -t java -t github repotests/java-sec-code -o bomresults/bom-java-sec-code-9.json
           bin/cdxgen.js -p -t java -exclude-type js repotests/java-sec-code -o bomresults/bom-java-sec-code-10.json
         shell: bash

diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.yml
@@ -58,6 +58,7 @@ jobs:
           SDKMAN_DIR: /home/runner/.sdkman
           CDXGEN_DEBUG_MODE: debug
           CDXGEN_LOG: /home/runner/work/new_snapshots/generate.log
+          PREFER_MAVEN_DEPS_TREE: false
         run: |
           mkdir /home/runner/work/new_snapshots
           python test/diff/generate.py

diff --git a/README.md b/README.md
@@ -33,6 +33,7 @@ Our philosophy:
 - Precision: Try using multiple techniques to improve precision, even if it takes extra time.
 - Personas: Cater to the needs of a range of personas such as security researchers, compliance auditors, developers, and SOC.
 - Lifecycle: Support BOM generation for various product lifecycles.
+- Machine Learning: Optimize the generated data for Machine Learning (ML) purposes by considering the various model properties.
 
 ## Documentation
 
@@ -116,6 +117,7 @@ Options:
       --deep                   Perform deep searches for components. Useful while scanning C/C++ apps, live OS and oci i
                                mages.                                                                          [boolean]
       --server-url             Dependency track url. Eg: https://deptrack.cyclonedx.io
+      --skip-dt-tls-check      Skip TLS certificate check when calling Dependency-Track.      [boolean] [default: false]
       --api-key                Dependency track api key
       --project-group          Dependency track project group
       --project-name           Dependency track project name. Default use the directory name
@@ -137,19 +139,19 @@ Options:
       --validate               Validate the generated SBOM using json schema. Defaults to true. Pass --no-validate to di
                                sable.                                                          [boolean] [default: true]
       --evidence               Generate SBOM with evidence for supported languages.           [boolean] [default: false]
-      --spec-version           CycloneDX Specification version to use. Defaults to 1.5           [number] [default: 1.5]
+      --spec-version           CycloneDX Specification version to use. Defaults to 1.6           [number] [default: 1.6]
       --filter                 Filter components containing this word in purl or component.properties.value. Multiple va
                                lues allowed.                                                                     [array]
       --only                   Include components only containing this word in purl. Useful to generate BOM with first p
                                arty components alone. Multiple values allowed.                                   [array]
       --author                 The person(s) who created the BOM. Set this value if you're intending the modify the BOM
                                and claim authorship.                               [array] [default: "OWASP Foundation"]
       --profile                BOM profile to use for generation. Default generic.
-  [choices: "appsec", "research", "operational", "threat-modeling", "license-compliance", "generic"] [default: "generic"
-                                                                                                                       ]
+  [choices: "appsec", "research", "operational", "threat-modeling", "license-compliance", "generic", "machine-learning",
+                                                                  "ml", "deep-learning", "ml-deep"] [default: "generic"]
       --exclude                Additional glob pattern(s) to ignore                                              [array]
-      --include-formulation    Generate formulation section with git metadata and build tools. Defaults to true. Invoke
-                               with --no-include-formulation to disable.                       [boolean] [default: true]
+      --include-formulation    Generate formulation section with git metadata and build tools. Defaults to false.
+                                                                                              [boolean] [default: false]
       --include-crypto         Include crypto libraries as components.                        [boolean] [default: false]
       --standard               The list of standards which may consist of regulations, industry or organizational-specif
                                ic standards, maturity models, best practices, or any other requirements which can be eva

diff --git a/bin/cdxgen.js b/bin/cdxgen.js
@@ -235,6 +235,10 @@ const args = yargs(hideBin(process.argv))
       "threat-modeling",
       "license-compliance",
       "generic",
+      "machine-learning",
+      "ml",
+      "deep-learning",
+      "ml-deep",
     ],
   })
   .option("lifecycle", {
@@ -414,6 +418,22 @@ const applyAdvancedOptions = (options) => {
     case "license-compliance":
       process.env.FETCH_LICENSE = "true";
       break;
+    case "machine-learning":
+    case "ml":
+      process.env.FETCH_LICENSE = "true";
+      options.deep = true;
+      options.evidence = false;
+      options.includeCrypto = false;
+      options.installDeps = true;
+      break;
+    case "deep-learning":
+    case "ml-deep":
+      process.env.FETCH_LICENSE = "true";
+      options.deep = true;
+      options.evidence = true;
+      options.includeCrypto = true;
+      options.installDeps = true;
+      break;
     default:
       break;
   }
@@ -699,6 +719,8 @@ const checkPermissions = (filePath) => {
         evinseOptions,
       );
       bomNSData.bomJson = evinseJson;
+      // Redo post processing with evinse data
+      bomNSData = postProcess(bomNSData, options);
       if (options.print && evinseJson) {
         printOccurrences(evinseJson);
         printCallStack(evinseJson);

diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -4,7 +4,7 @@ LABEL maintainer="cyclonedx" \
       org.opencontainers.image.authors="Prabhu Subramanian <[email protected]>" \
       org.opencontainers.image.source="https://github.com/cyclonedx/cdxgen" \
       org.opencontainers.image.url="https://github.com/cyclonedx/cdxgen" \
-      org.opencontainers.image.version="10.11.x" \
+      org.opencontainers.image.version="11.0.x" \
       org.opencontainers.image.vendor="cyclonedx" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cdxgen" \

diff --git a/ci/Dockerfile-bun b/ci/Dockerfile-bun
@@ -4,7 +4,7 @@ LABEL maintainer="cyclonedx" \
       org.opencontainers.image.authors="Prabhu Subramanian <[email protected]>" \
       org.opencontainers.image.source="https://github.com/cyclonedx/cdxgen" \
       org.opencontainers.image.url="https://github.com/cyclonedx/cdxgen" \
-      org.opencontainers.image.version="10.11.x" \
+      org.opencontainers.image.version="11.0.x" \
       org.opencontainers.image.vendor="cyclonedx" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cdxgen" \

diff --git a/ci/Dockerfile-deno b/ci/Dockerfile-deno
@@ -4,7 +4,7 @@ LABEL maintainer="cyclonedx" \
       org.opencontainers.image.authors="Prabhu Subramanian <[email protected]>" \
       org.opencontainers.image.source="https://github.com/cyclonedx/cdxgen" \
       org.opencontainers.image.url="https://github.com/cyclonedx/cdxgen" \
-      org.opencontainers.image.version="10.11.x" \
+      org.opencontainers.image.version="11.0.x" \
       org.opencontainers.image.vendor="cyclonedx" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cdxgen" \

diff --git a/ci/Dockerfile-ppc64 b/ci/Dockerfile-ppc64
@@ -4,7 +4,7 @@ LABEL maintainer="cyclonedx" \
       org.opencontainers.image.authors="Prabhu Subramanian <[email protected]>" \
       org.opencontainers.image.source="https://github.com/cyclonedx/cdxgen" \
       org.opencontainers.image.url="https://github.com/cyclonedx/cdxgen" \
-      org.opencontainers.image.version="10.11.x" \
+      org.opencontainers.image.version="11.0.x" \
       org.opencontainers.image.vendor="cyclonedx" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.title="cdxgen" \

diff --git a/ci/base-images/cdxgen/Dockerfile.rolling b/ci/base-images/cdxgen/Dockerfile.rolling
@@ -1,7 +1,6 @@
 FROM ghcr.io/cyclonedx/opensuse-lang:master
 
-ENV PREFER_MAVEN_DEPS_TREE=true \
-    CDXGEN_DEBUG_MODE=debug \
+ENV CDXGEN_DEBUG_MODE=debug \
     SCAN_DEBUG_MODE=debug \
     CDXGEN_IN_CONTAINER=true \
     PYTHONPATH=/opt/pypi

diff --git a/data/README.md b/data/README.md
@@ -22,4 +22,5 @@ Contents of data directory and their purpose.
 | wrapdb-releases.json  | Database of all available meson wraps. Generated using contrib/wrapdb.py.                                |
 | frameworks-list.json  | List of string fragments to categorize components into frameworks                                        |
 | crypto-oid.json       | Peter Gutmann's crypto oid [mapping](https://www.cs.auckland.ac.nz/~pgut001). GPL, BSD, or CC BY license |
-| glibc-stdlib.json     | Standard libraries that can be filtered out in C++ |
+| glibc-stdlib.json     | Standard libraries that can be filtered out in C++                                                       |
+| component-tags.json   | List of tags to extract from component description text for easy classification.                         |
diff --git a/data/component-tags.json b/data/component-tags.json
@@ -0,0 +1,160 @@
+{
+  "description": [
+    "sql",
+    "http",
+    "xml",
+    "web",
+    "security",
+    "database",
+    "json",
+    "yaml",
+    "validat",
+    "sanitizat",
+    "cloud",
+    "iam",
+    "auth",
+    "middleware",
+    "serializat",
+    "event",
+    "stream",
+    "rpc",
+    "socket",
+    "proto",
+    "resource",
+    "data",
+    "sensitive",
+    "template",
+    "log",
+    "logging",
+    "service",
+    "api",
+    "slf4j",
+    "parse",
+    "emit",
+    "jdbc",
+    "connect",
+    "pool",
+    "beans",
+    "transact",
+    "mysql",
+    "postgres",
+    "oracle",
+    "mongo",
+    "redis",
+    "splunk",
+    "stripe",
+    "payment",
+    "finance",
+    "currency",
+    "coin",
+    "monero",
+    "ssl",
+    "traffic",
+    "mvc",
+    "html",
+    "escape",
+    "unescape",
+    "rest",
+    "tomcat",
+    "jackson",
+    "hibernate",
+    "orm",
+    "aop",
+    "jwt",
+    "saml",
+    "token",
+    "tls",
+    "codec",
+    "cron",
+    "crypto",
+    "jce",
+    "certificate",
+    "developer",
+    "tools",
+    "autoconfigure",
+    "test",
+    "jsonpath",
+    "bytecode",
+    "mock",
+    "inject",
+    "comparators",
+    "transform",
+    "encode",
+    "decode",
+    "ldap",
+    "owasp",
+    "fileupload",
+    "beanshell",
+    "spel",
+    "mail",
+    "apacheds",
+    "jndi",
+    "ldif",
+    "jdbm",
+    "kerberos",
+    "oidc",
+    "oauth2",
+    "cli",
+    "binary",
+    "ml",
+    "ai",
+    "azure",
+    "gcp",
+    "terraform",
+    "redis",
+    "valkey",
+    "lint",
+    "bundle",
+    "object-persistence",
+    "text-to-image",
+    "translat",
+    "object-detect",
+    "mvc",
+    "framework",
+    "graph",
+    "templates",
+    "fastjson",
+    "simd",
+    "event-driven",
+    "productivity",
+    "annotations",
+    "typesafe",
+    "projections",
+    "performance",
+    "plugins",
+    "non-block",
+    "microsoft"
+  ],
+  "properties": [
+    "sql",
+    "http",
+    "xml",
+    "security",
+    "cloud",
+    "middleware",
+    "framework",
+    "bluetooth",
+    "wifi",
+    "wireless",
+    "driver",
+    "graphics",
+    "firmware",
+    "gyroscope",
+    "accelerometer",
+    "mobile",
+    "network",
+    "battery",
+    "matrix",
+    "thunderbolt",
+    "crypto",
+    "algorithm",
+    "encrypt",
+    "decrypt",
+    "registry",
+    "maps",
+    "payment",
+    "stripe",
+    "apple-pay",
+    "icloud"
+  ]
+}
diff --git a/deno.json b/deno.json
@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "10.11.0",
+  "version": "11.0.0",
   "exports": "./lib/cli/index.js",
   "compilerOptions": {
     "lib": ["deno.window"],
@@ -20,7 +20,7 @@
     "include": ["bin/", "*.js", "lib/**"],
     "exclude": ["test/", "data/", "contrib/"],
     "rules": {
-      "tags": ["recommended", "no-self-compare"],
+      "tags": ["recommended"],
       "exclude": ["no-unused-vars", "no-invalid-regexp"]
     }
   },

diff --git a/docs/ADVANCED.md b/docs/ADVANCED.md
@@ -41,7 +41,6 @@ Some package managers support filtering dependencies. For example, maven `depend
 Java maven example:
 
 ```shell
-export PREFER_MAVEN_DEPS_TREE=true
 export MVN_ARGS="-Dexcludes=:::*-SNAPSHOT"
 ```
 

diff --git a/docs/ENV.md b/docs/ENV.md
@@ -9,7 +9,7 @@ The following environment variables are available to configure the bom generatio
 | MVN_CMD                         | Set to override maven command                                                                                                                                                                                                                                                                                                       |
 | MVN_ARGS                        | Set to pass additional arguments such as profile or settings to maven                                                                                                                                                                                                                                                               |
 | MAVEN_HOME                      | Specify maven home                                                                                                                                                                                                                                                                                                                  |
-| PREFER_MAVEN_DEPS_TREE          | Use maven `dependency:tree` command instead of the cyclonedx maven plugin                                                                                                                                                                                                                                                           |
+| PREFER_MAVEN_DEPS_TREE          | Use maven `dependency:tree` command instead of the cyclonedx maven plugin. Defaults to true from v11. Set to false to use the cyclonedx-maven-plugin.                                                                                                                                                                               |
 | MAVEN_CENTRAL_URL               | Specify URL of Maven Central for metadata fetching (e.g. when private repo is used)                                                                                                                                                                                                                                                 |
 | ANDROID_MAVEN_URL               | Specify URL of Android Maven Repository for metadata fetching (e.g. when private repo is used)                                                                                                                                                                                                                                      |
 | BAZEL_ARGS                      | Additional arguments for Bazel command. Eg: --bazelrc=bazelrc.remote                                                                                                                                                                                                                                                                |

diff --git a/jsr.json b/jsr.json
@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "10.11.0",
+  "version": "11.0.0",
   "exports": "./lib/cli/index.js",
   "include": ["*.js", "lib/**", "bin/**", "data/**", "types/**"],
   "exclude": [

diff --git a/lib/evinser/evinser.js b/lib/evinser/evinser.js
@@ -1179,6 +1179,8 @@ export const createEvinseFile = (sliceArtefacts, options) => {
   const evinseOutFile = options.output;
   const bomJson = JSON.parse(fs.readFileSync(bomFile, "utf8"));
   const components = bomJson.components || [];
+  // Clear the existing annotations
+  bomJson.annotations = undefined;
   let occEvidencePresent = false;
   let csEvidencePresent = false;
   let servicesPresent = false;