Merge pull request #109 from openedx/cag/it

test: add integration testing

.github/workflows/integration-test.yml

@@ -0,0 +1,65 @@
+name: Integration Test
+
+on:
+  pull_request:
+
+jobs:
+
+  k8s:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        cluster:
+          - elasticsearch
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Kind
+        uses: helm/[email protected]
+        with:
+          install_only: true
+
+      - name: Start Kind
+        run: |
+          kind create cluster --config integration-test/cluster.yaml
+          kubectl config set-context --current --namespace=openedx-harmony
+          sudo echo "127.0.0.1 harmony.test" | sudo tee -a /etc/hosts
+
+      - name: Helm dependency add
+        run: |
+          helm dependency list charts/harmony-chart 2> /dev/null | tail +2 | awk '{ print "helm" " repo add " $1 " " $3 }' | while read cmd; do $cmd || true; done
+          kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.10.1/cert-manager.crds.yaml --namespace=harmony
+
+      - name: Helm dependency build
+        run: |
+          helm dependency update charts/harmony-chart
+          helm dependency build charts/harmony-chart
+
+      - name: Helm install
+        run: |
+          helm install harmony --namespace openedx-harmony --create-namespace -f integration-test/${{matrix.cluster}}/values.yaml charts/harmony-chart
+          kubectl rollout status deployment --timeout 300s
+
+      - name: Healthcheck
+        run: |
+          curl http://harmony.test/cluster-echo-test
+
+      - name: Boostrap cluster
+        run: |
+          bash integration-test/${{matrix.cluster}}/cluster.sh
+
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install openedx
+        run: |
+          export INSTALLATIONS=$(ls -d integration-test/${{matrix.cluster}}/*/)
+          export CI_ROOT=$(pwd)
+          for installation in $(ls -d integration-test/${{matrix.cluster}}/*/)
+          do
+            bash integration-test/environment.sh $installation
+          done

.gitignore

@@ -12,3 +12,5 @@ infra-*/secrets.auto.tfvars
 .terraform
 *secrets.auto.tfvars
 my-notes
+**/env/
+**/venv/

integration-test/cluster.yaml

@@ -0,0 +1,20 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    listenAddress: 127.0.0.1  # omit for 0.0.0.0
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    listenAddress: 127.0.0.1  # omit for 0.0.0.0
+    protocol: TCP
+

integration-test/elasticsearch/openedx-01/config.yml

@@ -0,0 +1,80 @@
+CMS_OAUTH2_SECRET: kWYPh15ecORZ75TvEqtzfghk
+DISCOVERY_INDEX_OVERRIDES:
+  course_discovery.apps.course_metadata.search_indexes.documents.course: openedx-01-owrj-course
+  course_discovery.apps.course_metadata.search_indexes.documents.course_run: openedx-01-owrj-course_run
+  course_discovery.apps.course_metadata.search_indexes.documents.learner_pathway: openedx-01-owrj-learner_pathway
+  course_discovery.apps.course_metadata.search_indexes.documents.person: openedx-01-owrj-person
+  course_discovery.apps.course_metadata.search_indexes.documents.program: openedx-01-owrj-program
+ELASTICSEARCH_HOST: harmony-search-cluster.openedx-harmony.svc.cluster.local
+ELASTICSEARCH_SCHEME: https
+ENABLE_HTTPS: true
+ENABLE_WEB_PROXY: false
+ID: K3vsruy7oOdTvOcKmhiCybNz
+JWT_RSA_PRIVATE_KEY: '-----BEGIN RSA PRIVATE KEY-----
+
+  MIIEowIBAAKCAQEA0IiZ4AR3Iz3PPzdjvCwG+sQT2Fqiq5/Z+SBudvJfhwoMKj6R
+
+  KNdmUcqsF+XotuQ6RRWaqmLv1INdlBjWlrhg7YdcJvJstIXE5Hs22M9R6Lth0jyB
+
+  159SEATsy9HuZBd10M7gG61LVoeamesn9U5Dd/0Fqfnb6PT2blYQ+SX24RYWV00K
+
+  +y1NujnK3NpVeEZGBNUk3xIxLT+W9EYarnGB1PdxQq+IQ56TTNDBWsuliTUkiXZN
+
+  CdDckefirDcUSbSh8VPlxuWNdIPe7CEPslUru9V93Vcs0NPIbXUSpEb5X5bq6gz9
+
+  ZfSn0bW0O1WwDR6GSiU9+KbdHxukAXGAqXIyiwIDAQABAoIBABEyoCS/ntxbAxtZ
+
+  Y+dyEWtJ6MG9i8/aoYfbBbEtQPU/6LaX2AcwaHb+Ch5NxC5HzonBrofhxpoAnCLp
+
+  aifI8e7KQdkmbLz15pcvQDE2O9uVOeE0Q0MMl8shai6kVA7mI73CP0JDvZmtGkdv
+
+  rikcDhK4XRNiZfaliSiaqXtasaxorILp6xXbWpFwpqiIhtvNgJjbgcKm05L4Ia0e
+
+  YWZ7LKfrM201oX0FyfgYxKk0lP6ZU74i961gBN1OLleW+5ikf6JXgJcRDmPg9+4E
+
+  JQTnj7EzlYg9+l39lltw/6yZaLRPU/o3GiXuLQWxJ7oYxaOFfWc00IjDWuXaCcni
+
+  Ftlv5CUCgYEA1TKaPR2/GE8EdgRZZ/nMPWgi+fptwGQfI4BtQkxZMOEQiVgjDdp/
+
+  t0l1dV3+HuG0VHyqfgI3OuO0SoIajZMaQEK4+DKQAIjTP5GR+vL+PltItZEYPl0y
+
+  ZULhJY5h6Rn5OtTaE/khQ4kGyu8VAIBhqSPYjIhxGpyLLKfys+APLQUCgYEA+mZJ
+
+  WpjZTfSqwKXMargsgn8uwXF2WgND9b/tqUkFAiwjkGvmXhpOzRUvq1mumFYphscs
+
+  5mdk+13WNRBmiHvJeRR9H3WcbSZTnVNfJ2QgY8b4uV7L8JetZs/jeF2X9Gkzi4Xa
+
+  UGs1N/DUGsZD1xPnTHCxGnt678oAbjIyLxVndk8CgYAON0uVm7AsXMh9XhjgGAYI
+
+  ajM62/oJhCOXaG7gptqjqb5Cij/8In+1pw5P/y58vKwbnhx4PomUQa658Sb4HcD4
+
+  W/qdgjGb94Ouj3ReeWl+mvOkN8KYFfrQSb9dk7rvoi2+QIcyIcCYEkEDsyGBYQ4i
+
+  CyPbaxWvopSdK63BVMVuSQKBgQDY8P3krEC5NZ0qw3rTizWk6iPMHU14txOOOvbh
+
+  kA3yz+MQISu5o/lZ3V0ri8uk2CGqT64rALGQxr/rwYXUAcuQR5wQQYmBV49h4vWx
+
+  3yHg0GZ11BfFHuRxCvP9OLmKXAr4esOmoPu+t7xMZjIoiWn/5DLMyua8EbtFvd+U
+
+  w2qcKQKBgBmTe8NzvMxKstgDcElCnpqFv1e2JVLofXIsrW+07r8qhPywfXIiloR9
+
+  JRaR1CjsA4ZsU5n+2fISx8RvZqnrYvsSLFrkW9U6iSkJJwSR0gTJb3sNaxpfv1uT
+
+  /GdzMnP1gM/V41wm28/LP6s9Z95tiwXjkHEBhIjYGbWi6DRz22jd
+
+  -----END RSA PRIVATE KEY-----'
+K8S_HARMONY_NAMESPACE: openedx-harmony
+K8S_HARMONY_SEARCH_CLUSTER_HTTP_AUTH: openedx-01:gXkYL5RNzzP3Zc6BHL1oJrKy
+K8S_HARMONY_SEARCH_CLUSTER_INDEX_PREFIX: openedx-01-owrj-
+K8S_NAMESPACE: openedx-01
+LMS_HOST: local.openedx.io
+MYSQL_ROOT_PASSWORD: c7SVvBJl
+OPENEDX_MYSQL_PASSWORD: M4CBFYO9
+OPENEDX_SECRET_KEY: dskQ9UXPKcsSi2molXmggxID
+PLUGINS:
+- indigo
+- k8s_harmony
+- mfe
+PLUGIN_INDEXES:
+- https://overhang.io/tutor/main
+RUN_ELASTICSEARCH: false

integration-test/elasticsearch/openedx-01/post-installation.sh

@@ -0,0 +1,2 @@
+# Run any arbitrary commands necessary to verify the installation is working
+echo "Make sure to change this script to verify your installation is tested correctly"

integration-test/elasticsearch/openedx-01/pre-init.sh

@@ -0,0 +1,8 @@
+export HARMONY_NAMESPACE=$(tutor config printvalue K8S_HARMONY_NAMESPACE)
+export INSTANCE_NAMESPACE=$(tutor config printvalue K8S_NAMESPACE)
+tutor harmony create-elasticsearch-user
+
+kubectl get secret "search-cluster-certificates-elasticsearch" -n "$HARMONY_NAMESPACE" -o "yaml" | \
+    grep -v '^\s*namespace:\s' | \
+    sed s/-elasticsearch//g |\
+    kubectl apply -n "$INSTANCE_NAMESPACE" --force -f -

integration-test/elasticsearch/openedx-01/requirements.txt

@@ -0,0 +1,2 @@
+../../../tutor-contrib-harmony-plugin
+tutor[full]<19

integration-test/elasticsearch/values.yaml

@@ -0,0 +1,50 @@
+# Disable HTTPS cert provisioning for testing with minikube
+cert-manager:
+   enabled: false
+
+ingress-nginx:
+  # Use ingress-nginx as a default controller.
+  enabled: true
+  controller:
+    # All these needed for local development
+    service:
+      type: NodePort
+    hostPort:
+      enabled: true
+    publishService:
+      enabled: false
+    extraArgs:
+      publish-status-address: localhost
+
+clusterDomain: harmony.test
+
+elasticsearch:
+  enabled: true
+
+   # TODO: move this to a separate PR
+   # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  volumeClaimTemplate:
+    resources:
+      requests:
+        storage: 8Gi
+  replicas: 1
+
+opensearch:
+  enabled: false
+
+   # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  persistence:
+    size: 8Gi
+
+prometheusstack:
+  enabled: true
+
+k8sdashboard:
+  enabled: false
+
+openfaas:
+  enabled: false

integration-test/environment.sh

@@ -0,0 +1,29 @@
+cd $1
+
+# Setup environment
+export TUTOR_PLUGINS_ROOT=$(pwd)/plugins/ export TUTOR_ROOT=$(pwd)
+python3 -m venv venv && source venv/bin/activate
+pip install -r $TUTOR_ROOT/requirements.txt
+
+# Start/init installation
+tutor config save
+tutor plugins enable k8s_harmony
+
+tutor k8s start
+
+echo "########################################"
+echo "#             Pre init                 #"
+echo "########################################"
+
+bash ./pre-init.sh
+
+tutor k8s init
+
+echo "########################################"
+echo "#             Post init                 #"
+echo "########################################"
+bash ./post-installation.sh
+
+# Go back
+deactivate
+cd $CI_ROOT

integration-test/template/openedx/config.yml

@@ -0,0 +1,9 @@
+K8S_HARMONY_NAMESPACE: openedx-harmony
+K8S_NAMESPACE: openedx-01
+LMS_HOST: local.openedx.io
+PLUGINS:
+- indigo
+- k8s_harmony
+- mfe
+PLUGIN_INDEXES:
+- https://overhang.io/tutor/main

integration-test/template/openedx/post-installation.sh

@@ -0,0 +1,2 @@
+# Run any arbitrary commands necessary to verify the installation is working
+echo "Make sure to change this script to verify your installation is tested correctly"

integration-test/template/openedx/pre-init.sh

@@ -0,0 +1,2 @@
+# Run any arbitrary commands necessary to setup the installation
+echo "Make sure to change this script to setup your installation"

integration-test/template/openedx/requirements.txt

@@ -0,0 +1,2 @@
+../../../tutor-contrib-harmony-plugin
+tutor[full]<19

integration-test/template/values.yaml

@@ -0,0 +1,50 @@
+clusterDomain: harmony.test
+
+# Disable HTTPS cert provisioning for testing with minikube
+cert-manager:
+   enabled: false
+
+ingress-nginx:
+  # Use ingress-nginx as a default controller.
+  enabled: true
+  controller:
+    # All these needed for local development
+    service:
+      type: NodePort
+    hostPort:
+      enabled: true
+    publishService:
+      enabled: false
+    extraArgs:
+      publish-status-address: localhost
+
+elasticsearch:
+  enabled: false
+
+   # TODO: move this to a separate PR
+   # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  volumeClaimTemplate:
+    resources:
+      requests:
+        storage: 8Gi
+  replicas: 1
+
+opensearch:
+  enabled: false
+
+   # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  persistence:
+    size: 8Gi
+
+prometheusstack:
+  enabled: false
+
+k8sdashboard:
+  enabled: false
+
+openfaas:
+  enabled: false

temp.txt

@@ -0,0 +1,34 @@
+# ingress nginx
+# cert manager
+# elastichsearch - openedx
+# opensearch - openedx
+# prometheusstack
+
+
+
+# steps
+# 1. Create a key for a CA
+openssl genrsa -des3 -out harmony.key 2048
+# 2. Generate a root certificate
+openssl req -x509 -new -nodes -key harmony.key -sha256 -days 1825 -out harmony.pem
+# 3. Install Certificate
+sudo cp harmony.pem /usr/local/share/ca-certificates/harmony.crt
+sudo update-ca-certificates
+# 4. Creating CA-signed certificates for dev site
+openssl genrsa -out harmony.test.key 2048
+# 4.1 Create CSR:
+openssl req -new -key harmony.test.key -out harmony.test.csr
+# 4.2 Sign certifcate
+openssl x509 -req -in harmony.test.csr -CA harmony.pem -CAkey harmony.key -CAcreateserial -out harmony.test.crt -days 825 -sha256 -extfile harmony.test.ext
+
+# Setup certificates
+sudo mkdir -p /etc/systemd/resolved.conf.d
+sudo tee /etc/systemd/resolved.conf.d/minikube.conf << EOF
+[Resolve]
+DNS=$(minikube ip)
+Domains=~harmony.test
+EOF
+sudo systemctl restart systemd-resolved
+
+# Enable certificate on minikube
+cp harmony.pem $HOME/.minikube/certs/harmony.pem

tutor-contrib-harmony-plugin/tutor_k8s_harmony_plugin/plugin.py

@@ -37,7 +37,7 @@ def is_plugin_loaded(plugin_name: str) -> bool:
         # instance.
         "ADDITIONAL_INGRESS_HOST_LIST": [],
         "ENABLE_SHARED_SEARCH_CLUSTER": False,
-        "DEPLOYMENT_REVISION_HISTORY_LIMIT": 10,
+        "DEPLOYMENT_REVISION_HISTORY_LIMIT": 10
     },
     "overrides": {
         # Don't use Caddy as a per-instance external web proxy, but do still use it