Merge pull request #654 from asfadmin/release-staging

Merge to master

.github/workflows/deploy-docs-website.yml

@@ -8,8 +8,8 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
       - run: pip install -r requirements/requirements-docs.txt

.github/workflows/lint.yml

@@ -6,8 +6,8 @@ jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.8
 
@@ -20,7 +20,7 @@ jobs:
   requirements:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Check requirements.in order
         run: |

.github/workflows/re-build.yml

@@ -36,11 +36,11 @@ jobs:
       - id: step1
         run: |
           VERSION=${GITHUB_REF_NAME#*.}
-          echo "::set-output name=version::$VERSION"
-          echo "::set-output name=code-zip::tea-code-build.$VERSION.zip"
-          echo "::set-output name=dependency-zip::tea-dependencylayer-build.$VERSION.zip"
-          echo "::set-output name=cloudformation-yaml::tea-cloudformation-build.$VERSION.yaml"
-          echo "::set-output name=terraform-zip::tea-terraform-build.$VERSION.zip"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+          echo "code-zip=tea-code-build.$VERSION.zip" >> $GITHUB_OUTPUT
+          echo "dependency-zip=tea-dependencylayer-build.$VERSION.zip" >> $GITHUB_OUTPUT
+          echo "cloudformation-yaml=tea-cloudformation-build.$VERSION.yaml" >> $GITHUB_OUTPUT
+          echo "terraform-zip=tea-terraform-build.$VERSION.zip" >> $GITHUB_OUTPUT
 
   # Build everything
   build-all:
@@ -49,7 +49,7 @@ jobs:
     needs: variables
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Load environment defaults
         run: cat .github/workflows/config-public/${{ inputs.environment }}.env >> $GITHUB_ENV
@@ -72,25 +72,25 @@ jobs:
         run: make build
 
       - name: Save dependency layer zip
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: dependency-layer
           path: dist/thin-egress-app-dependencies.zip
 
       - name: Save Lambda code
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: code
           path: dist/thin-egress-app-code.zip
 
       - name: Save CloudFormation yaml
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: cloudformation
           path: dist/thin-egress-app.yaml
 
       - name: Save Terraform zip
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: terraform
           path: dist/thin-egress-app-terraform.zip

.github/workflows/re-status.yml

@@ -38,12 +38,12 @@ jobs:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION || 'us-west-2' }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Load environment defaults
         run: cat .github/workflows/config-public/${{ inputs.environment }}.env >> $GITHUB_ENV
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - name: Upload success labels
         run: |
@@ -70,12 +70,12 @@ jobs:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION || 'us-west-2' }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Load environment defaults
         run: cat .github/workflows/config-public/${{ inputs.environment }}.env >> $GITHUB_ENV
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - name: Upload failure labels
         run: |

.github/workflows/re-test-e2e.yml

@@ -44,12 +44,12 @@ jobs:
       YAML_NAME: tea-cloudformation-run.${{ github.run_id }}.yaml
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Load environment defaults
         run: cat .github/workflows/config-public/${{ inputs.environment }}.env >> $GITHUB_ENV
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - name: Upload to AWS
         env:
@@ -153,6 +153,7 @@ jobs:
                   DomainCertArn="$DOMAIN_CERT_ARN" \
                   DomainName="$DOMAIN_NAME" \
                   EnableApiGatewayLogToCloudWatch="False" \
+                  EnableS3CredentialsEndpoint="True" \
                   JwtAlgo="RS256" \
                   JwtKeySecretName="$JWT_KEY_SECRET_NAME" \
                   LambdaCodeDependencyArchive="${CODE_PREFIX}${DEPENDENCY_NAME}" \

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
         env:
           RUN_TESTS: ${{ secrets.RUN_TESTS }}
         if: "${{ env.RUN_TESTS == 'true' }}"
-        run: echo "::set-output name=val::yes"
+        run: echo "val=yes" >> $GITHUB_OUTPUT
 
   test-e2e:
     needs:
@@ -60,12 +60,12 @@ jobs:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION || 'us-west-2' }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Load environment defaults
         run: cat .github/workflows/config-public/prod.env >> $GITHUB_ENV
 
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - name: Upload to public code bucket
         env:
@@ -92,7 +92,7 @@ jobs:
       - test-e2e
     if: success() || needs.test-e2e.result == 'skipped'
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
 
       - uses: actions/upload-release-asset@v1
         env:

.github/workflows/test-e2e.yml

@@ -23,7 +23,7 @@ jobs:
         env:
           RUN_TESTS: ${{ secrets.RUN_TESTS }}
         if: "${{ env.RUN_TESTS == 'true' }}"
-        run: echo "::set-output name=val::yes"
+        run: echo "val=yes" >> $GITHUB_OUTPUT
 
   test-e2e:
     needs:

.github/workflows/test.yml

@@ -11,8 +11,8 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v1
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       with:
         python-version: 3.8
 

Makefile

@@ -4,7 +4,8 @@ SOURCES := \
 	lambda/tea_bumper.py \
 	lambda/update_lambda.py
 
-RESOURCES := $(wildcard lambda/templates/*)
+HTML_TEMPLATES := $(wildcard lambda/templates/*.html)
+MD_TEMPLATES := $(wildcard lambda/templates/*.md)
 TERRAFORM := $(wildcard terraform/*)
 
 REQUIREMENTS_IN := $(wildcard requirements/*.in)
@@ -15,7 +16,9 @@ DIR := dist
 EMPTY := $(DIR)/empty
 # Temporary artifacts
 DIST_SOURCES := $(SOURCES:lambda/%=$(DIR)/code/%)
-DIST_RESOURCES := $(RESOURCES:lambda/%=$(DIR)/code/%)
+DIST_MD_RESOURCES := $(MD_TEMPLATES:lambda/%.md=$(DIR)/code/%.html)
+DIST_HTML_RESOURCES := $(HTML_TEMPLATES:lambda/%=$(DIR)/code/%)
+DIST_RESOURCES := $(DIST_HTML_RESOURCES) $(DIST_MD_RESOURCES)
 DIST_TERRAFORM := $(TERRAFORM:terraform/%=$(DIR)/terraform/%)
 
 BUCKET_MAP_OBJECT_KEY := DEFAULT
@@ -30,6 +33,9 @@ DOCKER := docker
 DOCKER_USER_ARG := --user "$(shell id -u):$(shell id -g)"
 DOCKER_COMMAND = $(DOCKER) run --rm $(DOCKER_USER_ARG) -v "$$PWD":/var/task $(DOCKER_ARGS)
 
+PYTHON := python3
+BUILD_VENV := $(DIR)/.venv
+
 #####################
 # Deployment Config #
 #####################
@@ -92,21 +98,31 @@ terraform: $(DIR)/thin-egress-app-terraform.zip
 clean:
 	rm -rf $(DIR)
 
+$(BUILD_VENV): requirements/requirements-make.txt
+	rm -rf $(BUILD_VENV)
+	$(PYTHON) -m venv $(BUILD_VENV)
+	$(BUILD_VENV)/bin/pip --cache-dir $(DIR)/.pip-cache/ install -r requirements/requirements-make.txt
+
 $(DIR)/thin-egress-app-dependencies.zip: requirements/requirements.txt $(REQUIREMENTS_DEPS)
 	rm -rf $(DIR)/python
 	@mkdir -p $(DIR)/python
 	$(DOCKER_LAMBDA_CI) build/dependency_builder.sh "$(DIR)/thin-egress-app-dependencies.zip" "$(DIR)"
 
+.SECONDARY: $(DIST_MD_RESOURCES)
+$(DIST_MD_RESOURCES): $(DIR)/code/%.html: lambda/%.md $(BUILD_VENV)
+	@mkdir -p $(@D)
+	$(BUILD_VENV)/bin/python scripts/render_md.py $< --output $@
+
 .SECONDARY: $(DIST_RESOURCES)
-$(DIST_RESOURCES): $(DIR)/code/%: lambda/%
+$(DIST_HTML_RESOURCES): $(DIR)/code/%: lambda/%
 	@mkdir -p $(@D)
 	cp $< $@
 
 .SECONDARY: $(DIST_SOURCES)
 $(DIST_SOURCES): $(DIR)/code/%: lambda/%
 	@mkdir -p $(@D)
 	cp $< $@
-	python3 scripts/sed.py -i $@ "<BUILD_ID>" "${BUILD_ID}"
+	$(PYTHON) scripts/sed.py -i $@ "<BUILD_ID>" "${BUILD_ID}"
 
 $(DIR)/thin-egress-app-code.zip: $(DIST_SOURCES) $(DIST_RESOURCES)
 	@mkdir -p $(DIR)/code
@@ -115,16 +131,16 @@ $(DIR)/thin-egress-app-code.zip: $(DIST_SOURCES) $(DIST_RESOURCES)
 $(DIR)/bucket-map.yaml:
 	cp config/bucket-map-template.yaml $@
 
-$(DIR)/thin-egress-app.yaml: cloudformation/thin-egress-app.yaml
+$(DIR)/thin-egress-app.yaml: cloudformation/thin-egress-app.yaml.j2 $(BUILD_VENV)
 	@mkdir -p $(DIR)
-	cp cloudformation/thin-egress-app.yaml $(DIR)/thin-egress-app.yaml
-ifdef CF_DEFAULT_CODE_BUCKET
-	python3 scripts/sed.py -i $(DIR)/thin-egress-app.yaml "asf.public.code" "${CF_DEFAULT_CODE_BUCKET}"
-endif
-	python3 scripts/sed.py -i $(DIR)/thin-egress-app.yaml "<DEPENDENCY_ARCHIVE_PATH_FILENAME>" "${CF_DEFAULT_DEPENDENCY_ARCHIVE_KEY}"
-	python3 scripts/sed.py -i $(DIR)/thin-egress-app.yaml "<CODE_ARCHIVE_PATH_FILENAME>" "${CF_DEFAULT_CODE_ARCHIVE_KEY}"
-	python3 scripts/sed.py -i $(DIR)/thin-egress-app.yaml "<BUILD_ID>" "${CF_BUILD_VERSION}"
-	python3 scripts/sed.py -i $(DIR)/thin-egress-app.yaml "^Description:.*" 'Description: "${CF_DESCRIPTION}"'
+	$(BUILD_VENV)/bin/python scripts/render_cf.py \
+		cloudformation/thin-egress-app.yaml.j2 \
+		--output $(DIR)/thin-egress-app.yaml \
+		--code-bucket "$(CF_DEFAULT_CODE_BUCKET)" \
+		--dependency-archive-key "$(CF_DEFAULT_DEPENDENCY_ARCHIVE_KEY)" \
+		--code-archive-key "$(CF_DEFAULT_CODE_ARCHIVE_KEY)" \
+		--build-version "$(CF_BUILD_VERSION)" \
+		--description "$(CF_DESCRIPTION)"
 
 .SECONDARY: $(DIST_TERRAFORM)
 $(DIST_TERRAFORM): $(DIR)/%: %
@@ -201,7 +217,7 @@ $(EMPTY)/.deploy-stack: $(DIR)/thin-egress-app.yaml $(EMPTY)/.deploy-dependencie
 					AuthBaseUrl=$(URS_URL) \
 					ConfigBucket=$(CONFIG_BUCKET) \
 					LambdaCodeS3Bucket=$(CODE_BUCKET) \
-					PermissionsBoundaryName= \
+					PermissionsBoundaryName=$(PERMISSION_BOUNDARY_NAME) \
 					BucketnamePrefix=$(BUCKETNAME_PREFIX) \
 					DownloadRoleArn="" \
 					DownloadRoleInRegionArn="" \
@@ -210,10 +226,11 @@ $(EMPTY)/.deploy-stack: $(DIR)/thin-egress-app.yaml $(EMPTY)/.deploy-dependencie
 					Loglevel=DEBUG \
 					Logtype=$(LOG_TYPE) \
 					Maturity=DEV \
-					PrivateVPC= \
-					VPCSecurityGroupIDs= \
-					VPCSubnetIDs= \
+					PrivateVPC=$(PRIVATE_VPC) \
+					VPCSecurityGroupIDs=$(VPC_SECURITY_GROUP_IDS) \
+					VPCSubnetIDs=$(VPC_SUBNET_IDS) \
 					EnableApiGatewayLogToCloudWatch="False" \
+					EnableS3CredentialsEndpoint="True" \
 					DomainName=$(DOMAIN_NAME-"") \
 					DomainCertArn=$(DOMAIN_CERT_ARN-"") \
 					CookieDomain=$(COOKIE_DOMAIN-"") \

Makefile.config.example

@@ -42,6 +42,16 @@ URS_URL := https://uat.urs.earthdata.nasa.gov
 # Logging style, either `flat` for human readable or `json` for machine readable
 LOG_TYPE := flat
 
+########################
+# For NGAP Deployments #
+########################
+# In an NGAP environment you must uncomment the following line or the deployment will fail
+# PERMISSION_BOUNDARY_NAME := NGAPShRoleBoundary
+
+PRIVATE_VPC :=
+VPC_SECURITY_GROUP_IDS :=
+VPC_SUBNET_IDS :=
+
 ####################################
 # CloudFormation Template Defaults #
 ####################################