k8s: my current state of work

This commits my current progress in re-creating the docker-compose
pipeline for PeekabooAV in Kubernetes.

That includes deployments, services, and hard-coded config files for
each step in the pipeline, modeled after what was done in the
[pipeline](/sett17/peekabooav-installer/tree/pipeline).

The yamls for cortex, and the set-up job, are included, although cortex
does currently not work inside of Kubernetes, due to the missing docker
runner. There is an open [PR](/TheHive-Project/Cortex/pull/349) and
corresponding issue.

Except for above mentioned cortex, the pipeline is fully functional.
Meaning one can send an email to the postfix_tx deployment, which is
then sent to the postfix_rx deployment and then processed by rspamd and
Peekaboo.

This was tested and developed with microk8s and a single node.

k8s/cortex_own.yaml

@@ -0,0 +1,150 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: cortex
+  namespace: peekabooav-pipeline
+spec:
+  selector:
+    app: cortex
+  sessionAffinity: None
+  type: NodePort
+  ports:
+  - name: cortex
+    port: 9001
+    protocol: TCP
+    targetPort: 9001
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cortex
+  namespace: peekabooav-pipeline
+  labels:
+    app: cortex
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cortex
+  template:
+    metadata:
+      labels:
+        app: cortex
+    spec:
+      containers:
+      - name: cortex
+        image: thehiveproject/cortex:3.1.4
+        volumeMounts:
+        - name: application
+          mountPath: /etc/cortex/application.conf
+          subPath: application.conf
+        - name: analyzers
+          mountPath: /etc/cortex/analyzers.json
+          subPath: analyzers.json
+        env:
+        - name: CORTEX_ADMIN_PASSWORD
+          value: dikka
+        - name: analyzer_urls
+          value: /etc/cortex/analyzers.json
+        ports:
+        - containerPort: 9001
+          name: cortex
+          protocol: TCP
+      dnsPolicy: ClusterFirst
+      volumes:
+      - name: application
+        configMap:
+          name: cortex-application-conf
+      - name: analyzers
+        configMap:
+          name: cortex-analyzers-json
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cortex-application-conf
+  namespace: peekabooav-pipeline
+  selfLink: /api/v1/namespaces/peekabooav-pipeline/configmaps/cortex-application-conf
+data:
+  application.conf: |
+    auth.method.basic=true
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cortex-analyzers-json
+  namespace: peekabooav-pipeline
+  selfLink: /api/v1/namespaces/peekabooav-pipeline/configmaps/cortex-analyzers-json
+data:
+  analyzers.json: |
+    [
+      {
+        "name": "FileInfo",
+        "version": "8.0",
+        "author": "TheHive-Project",
+        "url": "https://github.com/TheHive-Project/Cortex-Analyzers",
+        "license": "AGPL-V3",
+        "description": "Parse files in several formats such as OLE and OpenXML to detect VBA macros, extract their source code, generate useful information on PE, PDF files...",
+        "dataTypeList": ["file"],
+        "baseConfig": "FileInfo",
+        "configurationItems": [
+          {
+            "name": "manalyze_enable",
+            "description": "Wether to enable manalyze submodule or not.",
+            "type": "boolean",
+            "required": true,
+            "multi": false,
+            "defaultValue": false
+          },
+          {
+            "name": "manalyze_enable_docker",
+            "description": "Use docker to run Manalyze. Can be used only if not using the docker image of FileInfo",
+            "type": "boolean",
+            "required": false,
+            "multi": false,
+            "defaultValue": false
+          },
+          {
+            "name": "manalyze_enable_binary",
+            "description": "Use local binary to run Manalyze. Need to compile it before!",
+            "type": "boolean",
+            "required": false,
+            "multi": false,
+            "defaultValue": true
+          },
+          {
+            "name": "manalyze_binary_path",
+            "description": "Path to the Manalyze binary that was compiled before. Keep the default value if using the docker image of FileInfo ",
+            "type": "string",
+            "required": false,
+            "multi": false,
+            "defaultValue": "/worker/Manalyze/bin/manalyze"
+          },
+          {
+            "name": "floss_enable",
+            "description": "Enable the use of FireEye FLARE FLOSS",
+            "type": "boolean",
+            "required": false,
+            "multi": false,
+            "default": false
+          },
+          {
+            "name": "floss_binary_path",
+            "description": "Path to the FLOSS binary.",
+            "type": "string",
+            "required": false,
+            "multi": false,
+            "default": "/usr/bin/floss"
+          },
+          {
+            "name": "floss_minimal_string_length",
+            "description": "Length of strings must be in order to be considered.",
+            "type": "number",
+            "required": false,
+            "multi": false,
+            "default": 4
+          }
+        ],
+        "dockerImage": "cortexneurons/fileinfo:8"
+      }
+    ]

k8s/cortex_setup.yaml

@@ -0,0 +1,49 @@
+# apiVersion: apps/v1
+# kind: Deployment
+# metadata:
+#   name: cortex-setup
+#   namespace: peekabooav-pipeline
+#   labels:
+#     app: cortex-setup
+# spec:
+#   replicas: 1
+#   selector:
+#     matchLabels:
+#       app: cortex-setup
+#   template:
+#     metadata:
+#       labels:
+#         app: cortex-setup
+#     spec:
+#       containers:
+#       - name: cortex-setup
+#         image: peekabooav_cortex_setup:local
+#         env:
+#         - name: ELASTIC_URL
+#           value: elasticsearch:9200
+#         - name: CORTEX_URL
+#           value: cortex:9001
+#         - name: PEEKABOO_CORTEX_API_TOKEN
+#           value: dikka
+#       dnsPolicy: ClusterFirst
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cortex-setup
+  namespace: peekabooav-pipeline
+spec:
+  template:
+    spec:
+      containers:
+      - name: cortex-setup
+        image: peekabooav_cortex_setup:local
+        env:
+          - name: ELASTIC_URL
+            value: elasticsearch:9200
+          - name: CORTEX_URL
+            value: cortex:9001
+          - name: PEEKABOO_CORTEX_API_TOKEN
+            value: dikka
+          - name: CORTEX_ADMIN_PASSWORD
+            value: dikka
+      restartPolicy: Never

k8s/elassticsearch.yaml

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: peekabooav-pipeline
+spec:
+  selector:
+    app: elasticsearch
+  sessionAffinity: None
+  type: NodePort
+  ports:
+  - name: elasticsearch
+    port: 9200
+    protocol: TCP
+    targetPort: 9200
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  namespace: peekabooav-pipeline
+  labels:
+    app: elasticsearch
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+      - name: elasticsearch
+        image: elasticsearch:7.16.2
+        env:
+        - name: http.host
+          value: "0.0.0.0"
+        - name: discovery.type
+          value: single-node
+        - name: script.allowed_types
+          value: inline
+        - name: thread_pool.search.queue_size
+          value: "100000"
+        - name: thread_pool.write.queue_size
+          value: "10000"
+        - name: ES_HEAP_SIZE
+          value: 1g
+        - name: xpack.security.enabled
+          value: "false"
+        - name: cluster.routing.allocation.disk.watermark.flood_stage
+          value: 99%
+        - name: TAKE_FILE_OWNERSHIP
+          value: "1"
+      dnsPolicy: ClusterFirst

k8s/mariadb.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mariadb
+  namespace: peekabooav-pipeline
+spec:
+  selector:
+    app: mariadb
+  sessionAffinity: None
+  type: NodePort
+  ports:
+  - name: mariadb
+    port: 3306
+    protocol: TCP
+    targetPort: 3306
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb
+  namespace: peekabooav-pipeline
+  labels:
+    app: mariadb
+    type: database
+spec:
+  replicas: 1
+  selector:
+      matchLabels:
+        app: mariadb
+  template:
+    metadata:
+      labels:
+        app: mariadb
+        type: database
+    spec:
+      containers:
+        - name: mariadb
+          image: mariadb:10.3
+          readinessProbe:
+            exec:
+              command:
+              - /usr/bin/mysql
+              - --user=peekaboo
+              - --password=peekaboo
+              - --execute
+              - "SHOW DATABASES;"
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          env:
+            - name: MARIADB_RANDOM_ROOT_PASSWORD
+              value: "true"
+            - name: MARIADB_DATABASE
+              value: peekaboo
+            - name: MARIADB_USER
+              value: peekaboo
+            - name: MARIADB_PASSWORD
+              value: peekaboo