From 95f0c9a9f0b8c3fe7d5a9460e0861265a9168b7b Mon Sep 17 00:00:00 2001 From: James Lott Date: Tue, 12 Sep 2023 17:03:36 -0400 Subject: [PATCH] feat: deploy web app to staging environment (#1560) --- .github/workflows/stg_web_client_merge.yml | 51 +++++++++++++++++++ .github/workflows/stg_web_svc_merge.yml | 51 +++++++++++++++++++ web/client/.gitignore | 3 +- web/client/src/routes/+page.svelte | 5 +- web/pipeline/prd.env | 2 + web/pipeline/stg.env | 7 +++ web/service/gradle.properties | 10 ++++ .../web/service/util/StorageHelper.java | 13 +++-- .../web/service/util/StorageHelperTest.java | 4 +- 9 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/stg_web_client_merge.yml create mode 100644 .github/workflows/stg_web_svc_merge.yml create mode 100644 web/pipeline/stg.env diff --git a/.github/workflows/stg_web_client_merge.yml b/.github/workflows/stg_web_client_merge.yml new file mode 100644 index 0000000000..92c82cb424 --- /dev/null +++ b/.github/workflows/stg_web_client_merge.yml @@ -0,0 +1,51 @@ +name: Web client staging CD + +on: + push: + branches: + - master + +jobs: + + web_cd: + name: Deploy web client + runs-on: ubuntu-latest + + env: + ENV_NAME : stg + CLOUDSDK_CORE_PROJECT : web-based-gtfs-validator + + steps: + + - uses: actions/checkout@v3 + with: + # We need to download all tags so that the axion-release-plugin + # can resolve the most recent version tag. + fetch-depth: 0 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + # We need a recent version of Java with jpackage included. + java-version: '17' + # We use the zulu distribution, which is an OpenJDK distro. + distribution: 'zulu' + + # for npm + - uses: actions/setup-node@v3 + with: + node-version: 16 + + - uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_WEB_VALIDATOR_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v1 + with: + version: '>= 390.0.0' + + - name: run gradle tasks + shell: bash + run: | + ENV_FILE=web/pipeline/${ENV_NAME}.env source web/pipeline/env-file.sh + ./gradlew ':web:client:webDeploy' diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml new file mode 100644 index 0000000000..e8c1b7b9e7 --- /dev/null +++ b/.github/workflows/stg_web_svc_merge.yml @@ -0,0 +1,51 @@ +name: Web service staging CD + +on: + push: + branches: + - master + +jobs: + + web_cd: + name: Deploy web service + runs-on: ubuntu-latest + + env: + ENV_NAME : stg + CLOUDSDK_CORE_PROJECT : web-based-gtfs-validator + + steps: + + - uses: actions/checkout@v3 + with: + # We need to download all tags so that the axion-release-plugin + # can resolve the most recent version tag. + fetch-depth: 0 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + # We need a recent version of Java with jpackage included. + java-version: '17' + # We use the zulu distribution, which is an OpenJDK distro. + distribution: 'zulu' + + # for npm + - uses: actions/setup-node@v3 + with: + node-version: 16 + + - uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.GCP_WEB_VALIDATOR_SA_KEY }} + + - uses: google-github-actions/setup-gcloud@v1 + with: + version: '>= 390.0.0' + + - name: run gradle tasks + shell: bash + run: | + ENV_FILE=web/pipeline/${ENV_NAME}.env source web/pipeline/env-file.sh + ./gradlew ':web:service:webDeploy' diff --git a/web/client/.gitignore b/web/client/.gitignore index 3d4439f65b..c88c4af9cb 100644 --- a/web/client/.gitignore +++ b/web/client/.gitignore @@ -8,4 +8,5 @@ node_modules !.env.example vite.config.js.timestamp-* vite.config.ts.timestamp-* -/static/rules.json +/notice_schema.json +rules.json diff --git a/web/client/src/routes/+page.svelte b/web/client/src/routes/+page.svelte index 5f1f9b266c..6e91ca4dd5 100644 --- a/web/client/src/routes/+page.svelte +++ b/web/client/src/routes/+page.svelte @@ -15,6 +15,7 @@ import { fly } from 'svelte/transition'; import { onMount, tick } from 'svelte'; import { quintOut } from 'svelte/easing'; + import { env } from '$env/dynamic/public'; /** * @typedef CreateJobParameters @@ -35,7 +36,7 @@ let showDocs = true; - const apiRoot = 'https://gtfs-validator-web-mbzoxaljzq-ue.a.run.app'; + const apiRoot = `${env.PUBLIC_CLIENT_API_ROOT}`; /** @type {HTMLInputElement} */ let fileInput; @@ -67,7 +68,7 @@ /** @type {HTMLDialogElement} */ let statusModal; - $: reportUrl = `https://gtfs-validator-results.mobilitydata.org/${jobId}/report.html`; + $: reportUrl = `${env.PUBLIC_CLIENT_REPORTS_ROOT}/${jobId}/report.html`; function clearErrors() { errors = []; diff --git a/web/pipeline/prd.env b/web/pipeline/prd.env index 6c8b8843f8..4dc233c107 100644 --- a/web/pipeline/prd.env +++ b/web/pipeline/prd.env @@ -3,3 +3,5 @@ WEB_BUILD_ENV=prd WEB_DEPLOY_CLIENT_BUCKET=gtfs-validator-web WEB_DEPLOY_SVC_IMAGE=gcr.io/web-based-gtfs-validator/gtfs-validator-web WEB_DEPLOY_SVC_CLOUDRUN=gtfs-validator-web +PUBLIC_CLIENT_API_ROOT=https://gtfs-validator-web-mbzoxaljzq-ue.a.run.app +PUBLIC_CLIENT_REPORTS_ROOT=https://gtfs-validator-results.mobilitydata.org diff --git a/web/pipeline/stg.env b/web/pipeline/stg.env new file mode 100644 index 0000000000..72148175e4 --- /dev/null +++ b/web/pipeline/stg.env @@ -0,0 +1,7 @@ +CLOUDSDK_CORE_PROJECT=web-based-gtfs-validator +WEB_BUILD_ENV=stg +WEB_DEPLOY_CLIENT_BUCKET=stg-gtfs-validator-web +WEB_DEPLOY_SVC_IMAGE=gcr.io/web-based-gtfs-validator/gtfs-validator-web +WEB_DEPLOY_SVC_CLOUDRUN=stg-gtfs-validator-web +PUBLIC_CLIENT_API_ROOT=https://stg-gtfs-validator-web-mbzoxaljzq-ue.a.run.app +PUBLIC_CLIENT_REPORTS_ROOT=https://staging-gtfs-validator-results.mobilitydata.org diff --git a/web/service/gradle.properties b/web/service/gradle.properties index f48a47add5..2870a44fa0 100644 --- a/web/service/gradle.properties +++ b/web/service/gradle.properties @@ -7,3 +7,13 @@ env.prd.svcAPMLicenseDst = newrelic/newrelic.yml env.prd.svcSecretPropertiesProvider = gcloud-secrets env.prd.svcSecretPropertiesSrc = web-validator-properties env.prd.svcSecretPropertiesDst = src/main/resources/application.properties + +env.stg.svcCredentialProvider = gcloud-secrets +env.stg.svcCredentialSrc = web-validator-sa-key +env.stg.svcCredentialDst = src/main/resources/web-based-gtfs-validator-a088ec5f045d.json +env.stg.svcAPMLicenseProvider = gcloud-secrets +env.stg.svcAPMLicenseSrc = web-validator-newrelic-license +env.stg.svcAPMLicenseDst = newrelic/newrelic.yml +env.stg.svcSecretPropertiesProvider = gcloud-secrets +env.stg.svcSecretPropertiesSrc = web-validator-properties +env.stg.svcSecretPropertiesDst = src/main/resources/application.properties diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java index 935a78ea36..9ae9015b12 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java @@ -19,13 +19,18 @@ /** Helper class for interacting with GCS. */ @Component public class StorageHelper { - public static final String JOB_INFO_BUCKET_NAME = "gtfs-validator-results"; + public String JOB_INFO_BUCKET_NAME = + System.getenv().getOrDefault("JOB_INFO_BUCKET_NAME", "gtfs-validator-results"); + static final String JOB_FILENAME_PREFIX = "job"; static final String JOB_FILENAME_SUFFIX = ".json"; public static final String JOB_FILENAME = JOB_FILENAME_PREFIX + JOB_FILENAME_SUFFIX; - public static final String TEMP_FOLDER_NAME = "gtfs-validator-temp"; - static final String USER_UPLOAD_BUCKET_NAME = "gtfs-validator-user-uploads"; - static final String RESULTS_BUCKET_NAME = "gtfs-validator-results"; + public static final String TEMP_FOLDER_NAME = + System.getenv().getOrDefault("TEMP_FOLDER_NAME", "gtfs-validator-temp"); + static final String USER_UPLOAD_BUCKET_NAME = + System.getenv().getOrDefault("USER_UPLOAD_BUCKET_NAME", "gtfs-validator-user-uploads"); + static final String RESULTS_BUCKET_NAME = + System.getenv().getOrDefault("RESULTS_BUCKET_NAME", "gtfs-validator-results"); static final String FILE_NAME = "gtfs-job.zip"; private final Logger logger = LoggerFactory.getLogger(StorageHelper.class); diff --git a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelperTest.java b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelperTest.java index 9ff3e8bcf9..d3dd5727fb 100644 --- a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelperTest.java +++ b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelperTest.java @@ -44,7 +44,7 @@ public void testSaveJobMetadata() throws Exception { byte[] bytes = byteArrayCaptor.getValue(); String expectedPath = jobMetadata.getJobId() + "/" + StorageHelper.JOB_FILENAME; BlobInfo expectedBlobInfo = - BlobInfo.newBuilder(BlobId.of(StorageHelper.JOB_INFO_BUCKET_NAME, expectedPath)) + BlobInfo.newBuilder(BlobId.of(storageHelper.JOB_INFO_BUCKET_NAME, expectedPath)) .setContentType("application/json") .build(); byte[] expectedBytes = mapper.writeValueAsString(jobMetadata).getBytes(); @@ -69,7 +69,7 @@ public void testGetJobMetadata() throws Exception { verify(storage, times(1)).get(blobIdCaptor.capture()); BlobId expectedBlobId = - BlobId.of(StorageHelper.JOB_INFO_BUCKET_NAME, StorageHelper.getJobInfoPath(testJobId)); + BlobId.of(storageHelper.JOB_INFO_BUCKET_NAME, StorageHelper.getJobInfoPath(testJobId)); assertEquals(expectedBlobId, blobIdCaptor.getValue()); assertEquals(expectedJson, mapper.writeValueAsString(actualJobMetadata));