remove agent_template.yaml from terraform generation (#129)

* remove agent_template.yaml from terraform generation * update terraform bootstrap file * urlescape the artifact url * remove install configurable and agent template creation * remove domino install from deploy test * add BASTION_IP to outputs * add bastion_ip to cdk tf main
dominodatalab · Feb 22, 2023 · d5b20dc · d5b20dc
1 parent 2635cf9
commit d5b20dc
Show file tree

Hide file tree

Showing 14 changed files with 64 additions and 559 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -94,7 +94,8 @@ jobs:
         for suffix in "" "-terraform"; do
         filename="domino-cdk$suffix-$DOMINO_CDK_VERSION.tar.gz"
         aws s3 cp --acl=public-read ./dist/$filename s3://domino-artifacts/cdk/$($DATEDIR)/$filename
-        echo "Artifact url: https://domino-artifacts.s3.amazonaws.com/cdk/$($DATEDIR)/$filename"
+        urlfile=$(python -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip()))' <<< "$filename")
+        echo "Artifact url: https://domino-artifacts.s3.amazonaws.com/cdk/$($DATEDIR)/$urlfile"
         done
     - name: Deploy CDK
       if: contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master'
@@ -106,61 +107,13 @@ jobs:
       run: |
         docker login -u $REGISTRY_USERNAME -p $REGISTRY_PASSWORD quay.io
         cdk deploy --require-approval never --outputs-file outputs.json
-    - name: Prepare Domino install
-      if: contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master'
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_ACCESS_KEY }}
-      run: |
-        $(jq -r ".[].ekskubeconfigcmd" outputs.json) --kubeconfig ./kubeconfig
-        jq -r ".[].agentconfig" outputs.json > agent_template.yaml
-        docker run --rm -v $(pwd):/cdk $DEPLOYER_IMAGE python -m fleetcommand_agent init --full -t /cdk/agent_template.yaml -f /cdk/domino.yml
-    - name: Install Domino
-      if: contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master'
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_ACCESS_KEY }}
-        KUBECONFIG: ./kubeconfig
-        LOG_DIR: k8s-cluster-state
-      run: |
-        docker run --rm -v $(pwd):/cdk -v $(pwd)/agent_logs:/domino-deployer/logs -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e KUBECONFIG=/cdk/kubeconfig $DEPLOYER_IMAGE python -m fleetcommand_agent run -f /cdk/domino.yml
-    - name: Collect diagnostic data
-      if: always() && (contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master')
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_ACCESS_KEY }}
-        KUBECONFIG: ./kubeconfig
-        LOG_DIR: k8s-cluster-state
-      run: |
-        set +e
-        curl -Lo /usr/local/bin/kubectl  "https://dl.k8s.io/release/v1.23.6/bin/linux/amd64/kubectl"
-        for ns in domino-platform domino-compute domino-system kube-system; do
-          mkdir -p $LOG_DIR/$ns
-          kubectl -n $ns get ing -o yaml > $LOG_DIR/$ns/ingress.txt
-          kubectl -n $ns get po -o yaml > $LOG_DIR/$ns/pods.txt
-          kubectl -n $ns describe po > $LOG_DIR/$ns/pods-described.txt
-          kubectl -n $ns get pvc -o yaml > $LOG_DIR/$ns/pvcs.txt
-          kubectl -n $ns get svc -o yaml > $LOG_DIR/$ns/svcs.txt
-          kubectl -n $ns describe svc > $LOG_DIR/$ns/svcs-described.txt
-          kubectl -n $ns get events > $LOG_DIR/$ns/events.txt
-        done
-        kubectl get pv -o yaml > $LOG_DIR/pvs.txt
-        kubectl get no -o yaml > $LOG_DIR/nodes.txt
-        kubectl describe no > $LOG_DIR/nodes-described.txt
     - name: Upload diagnostic data
       if: always() && (contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master')
       uses: actions/upload-artifact@v2
       with:
         name: Diagnostic Data
         path: ./
         retention-days: 14
-    - name: Uninstall Domino
-      if: always() && (contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master')
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.DELTA_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.DELTA_ACCESS_KEY }}
-      run: |
-        docker run --rm -v $(pwd):/cdk -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY -e KUBECONFIG=/cdk/kubeconfig $DEPLOYER_IMAGE python -m fleetcommand_agent destroy -f /cdk/domino.yml
     - name: Destroy CDK
       if: always() && (contains(github.event.pull_request.labels.*.name, 'deploy-test') || github.ref == 'refs/heads/master')
       env:

diff --git a/cdk/domino_cdk/agent.py b/cdk/domino_cdk/agent.py
diff --git a/cdk/domino_cdk/config/__init__.py b/cdk/domino_cdk/config/__init__.py
@@ -7,7 +7,6 @@
 from domino_cdk.config.base import DominoCDKConfig
 from domino_cdk.config.efs import EFS
 from domino_cdk.config.eks import EKS
-from domino_cdk.config.install import Install
 from domino_cdk.config.route53 import Route53
 from domino_cdk.config.s3 import S3
 from domino_cdk.config.util import IngressRule

diff --git a/cdk/domino_cdk/config/base.py b/cdk/domino_cdk/config/base.py
@@ -10,7 +10,6 @@
 from domino_cdk.config.acm import ACM
 from domino_cdk.config.efs import EFS
 from domino_cdk.config.eks import EKS
-from domino_cdk.config.install import Install
 from domino_cdk.config.route53 import Route53
 from domino_cdk.config.s3 import S3
 from domino_cdk.config.util import from_loader
@@ -41,8 +40,6 @@ class DominoCDKConfig:
     s3: Optional[S3] = None
     acm: Optional[ACM] = None
 
-    install: Optional[Install] = None
-
     @field_property(tags)
     def get_tags(self) -> Dict[str, str]:
         return {**unwrap_property(self).tags, **{"domino-deploy-id": self.name}}
@@ -68,9 +65,8 @@ def from_0_0_0(c: dict):
         if efs is not None:
             efs = EFS.from_0_0_0(efs)
 
-        install = c.pop("install", None)
-        if install is not None:
-            install = Install.from_0_0_0(install)
+        # Install is no longer supported
+        c.pop("install", None)
 
         acm = c.pop("acm", None)
         if acm is not None:
@@ -90,7 +86,6 @@ def from_0_0_0(c: dict):
                 route53=route53,
                 eks=EKS.from_0_0_0(c.pop("eks")),
                 s3=s3,
-                install=install,
                 acm=acm,
             ),
             c,
@@ -110,9 +105,8 @@ def from_0_0_1(c: dict):
         if efs is not None:
             efs = EFS.from_0_0_0(efs)
 
-        install = c.pop("install", None)
-        if install is not None:
-            install = Install.from_0_0_1(install)
+        # Install is no longer supported
+        c.pop("install", None)
 
         acm = c.pop("acm", None)
         if acm is not None:
@@ -132,7 +126,6 @@ def from_0_0_1(c: dict):
                 route53=route53,
                 eks=EKS.from_0_0_1(c.pop("eks")),
                 s3=s3,
-                install=install,
                 acm=acm,
             ),
             c,