From 040467b39fed6ef0666b75c5fc4d38b7c6d4ba7e Mon Sep 17 00:00:00 2001 From: Reuven Gonzales Date: Fri, 5 Apr 2024 15:47:51 -0700 Subject: [PATCH] Update Hasura to use dbt metadata (#1188) * Call dbt from inside the hasura table meta generator * Update workflows --- .github/workflows/deploy-hasura.yml | 14 ++++ .../workflows/refresh-test-credentials.yml | 7 ++ ...project.yaml => deployers_by_project.yaml} | 2 +- ...tion.yaml => event_totals_by_project.yaml} | 2 +- ...oject_arbitrum.yaml => pm_dev_months.yaml} | 2 +- ...ion_arbitrum.yaml => pm_new_contribs.yaml} | 2 +- .../cloudsql/tables/repos_by_project.yaml | 22 ++++++ .../databases/cloudsql/tables/tables.yaml | 49 +++++++------ apps/hasura/package.json | 4 +- apps/hasura/src/genTables.ts | 68 ++++++++++++++++--- 10 files changed, 134 insertions(+), 38 deletions(-) rename apps/hasura/metadata/databases/cloudsql/tables/{github_metrics_by_project.yaml => deployers_by_project.yaml} (92%) rename apps/hasura/metadata/databases/cloudsql/tables/{github_metrics_by_collection.yaml => event_totals_by_project.yaml} (91%) rename apps/hasura/metadata/databases/cloudsql/tables/{onchain_metrics_by_project_arbitrum.yaml => pm_dev_months.yaml} (90%) rename apps/hasura/metadata/databases/cloudsql/tables/{onchain_metrics_by_collection_arbitrum.yaml => pm_new_contribs.yaml} (89%) create mode 100644 apps/hasura/metadata/databases/cloudsql/tables/repos_by_project.yaml diff --git a/.github/workflows/deploy-hasura.yml b/.github/workflows/deploy-hasura.yml index 64f4b3ecd..2a90b6ba7 100644 --- a/.github/workflows/deploy-hasura.yml +++ b/.github/workflows/deploy-hasura.yml @@ -4,6 +4,7 @@ env: HASURA_GRAPHQL_ADMIN_SECRET: ${{ secrets.HASURA_GRAPHQL_ADMIN_SECRET }} HASURA_GRAPHQL_ENDPOINT: ${{ vars.HASURA_GRAPHQL_ENDPOINT }} HASURA_GRAPHQL_DATABASE_URL: ${{ secrets.HASURA_GRAPHQL_DATABASE_URL }} + GOOGLE_CREDENTIALS_JSON: ${{ vars.GOOGLE_TEST_DUMMY_CREDENTIALS_JSON }} # Trigger the workflow when: on: @@ -27,6 +28,7 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 1 + - name: Setup pnpm uses: pnpm/action-setup@v2 with: @@ -34,11 +36,23 @@ jobs: run_install: | - recursive: true args: [--frozen-lockfile, --strict-peer-dependencies] + - name: Set up Node.js 20 uses: actions/setup-node@v3 with: cache: "pnpm" node-version: "20.x" + + - name: Login to google + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GOOGLE_BQ_ADMIN_CREDENTIALS_JSON }}' + create_credentials_file: true + + - name: Setup dbt profile + run: | + bash .github/scripts/create-dbt-profile.sh ${GOOGLE_APPLICATION_CREDENTIALS} + - name: Build run: pnpm build:hasura - name: Deploy diff --git a/.github/workflows/refresh-test-credentials.yml b/.github/workflows/refresh-test-credentials.yml index d5ef23b71..5658caee9 100644 --- a/.github/workflows/refresh-test-credentials.yml +++ b/.github/workflows/refresh-test-credentials.yml @@ -63,6 +63,13 @@ jobs: cd ops/external-prs && bash scripts/rotate-service-account.sh bigquery-admin@oso-pull-requests.iam.gserviceaccount.com bigquery-admin.json && pnpm tools refresh-gcp-credentials ${{ github.repository }} external-prs-app bigquery-admin.json GOOGLE_BQ_ADMIN_CREDENTIALS_JSON + + - name: Refresh credentials for the bigquery-admin user on the deploy environment + shell: bash + run: | + cd ops/external-prs && + bash scripts/rotate-service-account.sh bigquery-admin@oso-pull-requests.iam.gserviceaccount.com bigquery-admin.json && + pnpm tools refresh-gcp-credentials ${{ github.repository }} deploy bigquery-admin.json GOOGLE_BQ_ADMIN_CREDENTIALS_JSON rebuild-docker-public-vars: name: rebuild-docker-public-vars diff --git a/apps/hasura/metadata/databases/cloudsql/tables/github_metrics_by_project.yaml b/apps/hasura/metadata/databases/cloudsql/tables/deployers_by_project.yaml similarity index 92% rename from apps/hasura/metadata/databases/cloudsql/tables/github_metrics_by_project.yaml rename to apps/hasura/metadata/databases/cloudsql/tables/deployers_by_project.yaml index 4e7430793..a5adc7da0 100644 --- a/apps/hasura/metadata/databases/cloudsql/tables/github_metrics_by_project.yaml +++ b/apps/hasura/metadata/databases/cloudsql/tables/deployers_by_project.yaml @@ -1,5 +1,5 @@ table: - name: github_metrics_by_project + name: deployers_by_project schema: public select_permissions: - role: anonymous diff --git a/apps/hasura/metadata/databases/cloudsql/tables/github_metrics_by_collection.yaml b/apps/hasura/metadata/databases/cloudsql/tables/event_totals_by_project.yaml similarity index 91% rename from apps/hasura/metadata/databases/cloudsql/tables/github_metrics_by_collection.yaml rename to apps/hasura/metadata/databases/cloudsql/tables/event_totals_by_project.yaml index c251df5df..0dfebc311 100644 --- a/apps/hasura/metadata/databases/cloudsql/tables/github_metrics_by_collection.yaml +++ b/apps/hasura/metadata/databases/cloudsql/tables/event_totals_by_project.yaml @@ -1,5 +1,5 @@ table: - name: github_metrics_by_collection + name: event_totals_by_project schema: public select_permissions: - role: anonymous diff --git a/apps/hasura/metadata/databases/cloudsql/tables/onchain_metrics_by_project_arbitrum.yaml b/apps/hasura/metadata/databases/cloudsql/tables/pm_dev_months.yaml similarity index 90% rename from apps/hasura/metadata/databases/cloudsql/tables/onchain_metrics_by_project_arbitrum.yaml rename to apps/hasura/metadata/databases/cloudsql/tables/pm_dev_months.yaml index 73b38d2b0..fa8c301d9 100644 --- a/apps/hasura/metadata/databases/cloudsql/tables/onchain_metrics_by_project_arbitrum.yaml +++ b/apps/hasura/metadata/databases/cloudsql/tables/pm_dev_months.yaml @@ -1,5 +1,5 @@ table: - name: onchain_metrics_by_project_arbitrum + name: pm_dev_months schema: public select_permissions: - role: anonymous diff --git a/apps/hasura/metadata/databases/cloudsql/tables/onchain_metrics_by_collection_arbitrum.yaml b/apps/hasura/metadata/databases/cloudsql/tables/pm_new_contribs.yaml similarity index 89% rename from apps/hasura/metadata/databases/cloudsql/tables/onchain_metrics_by_collection_arbitrum.yaml rename to apps/hasura/metadata/databases/cloudsql/tables/pm_new_contribs.yaml index 1bc2707fb..dce16eb6e 100644 --- a/apps/hasura/metadata/databases/cloudsql/tables/onchain_metrics_by_collection_arbitrum.yaml +++ b/apps/hasura/metadata/databases/cloudsql/tables/pm_new_contribs.yaml @@ -1,5 +1,5 @@ table: - name: onchain_metrics_by_collection_arbitrum + name: pm_new_contribs schema: public select_permissions: - role: anonymous diff --git a/apps/hasura/metadata/databases/cloudsql/tables/repos_by_project.yaml b/apps/hasura/metadata/databases/cloudsql/tables/repos_by_project.yaml new file mode 100644 index 000000000..6199dc918 --- /dev/null +++ b/apps/hasura/metadata/databases/cloudsql/tables/repos_by_project.yaml @@ -0,0 +1,22 @@ +table: + name: repos_by_project + schema: public +select_permissions: + - role: anonymous + permission: + columns: "*" + filter: {} + allow_aggregations: false + comment: "" + - role: user + permission: + columns: "*" + filter: {} + allow_aggregations: false + comment: "" + - role: developer + permission: + columns: "*" + filter: {} + allow_aggregations: true + comment: "" diff --git a/apps/hasura/metadata/databases/cloudsql/tables/tables.yaml b/apps/hasura/metadata/databases/cloudsql/tables/tables.yaml index d0ce7877e..a8e4af6cd 100644 --- a/apps/hasura/metadata/databases/cloudsql/tables/tables.yaml +++ b/apps/hasura/metadata/databases/cloudsql/tables/tables.yaml @@ -1,31 +1,36 @@ -- "!include onchain_metrics_by_collection.yaml" -- "!include onchain_metrics_by_project.yaml" +- "!include artifacts.yaml" +- "!include artifacts_by_project.yaml" +- "!include code_metrics_by_collection.yaml" +- "!include code_metrics_by_project.yaml" +- "!include collections.yaml" +- "!include deployers_by_project.yaml" +- "!include event_totals_by_project.yaml" - "!include event_types.yaml" -- "!include first_contribution_to_project.yaml" -- "!include last_contribution_to_project.yaml" -- "!include users_monthly_to_project.yaml" -- "!include events_weekly_from_artifact.yaml" -- "!include events_weekly_from_collection.yaml" -- "!include events_weekly_from_project.yaml" -- "!include events_weekly_to_artifact.yaml" -- "!include events_weekly_to_collection.yaml" -- "!include events_weekly_to_project.yaml" -- "!include events_monthly_from_artifact.yaml" -- "!include events_monthly_from_collection.yaml" -- "!include events_monthly_from_project.yaml" -- "!include events_monthly_to_artifact.yaml" -- "!include events_monthly_to_collection.yaml" -- "!include events_monthly_to_project.yaml" - "!include events_daily_from_artifact.yaml" - "!include events_daily_from_collection.yaml" - "!include events_daily_from_project.yaml" - "!include events_daily_to_artifact.yaml" - "!include events_daily_to_collection.yaml" - "!include events_daily_to_project.yaml" -- "!include artifacts.yaml" -- "!include artifacts_by_project.yaml" -- "!include collections.yaml" +- "!include events_monthly_from_artifact.yaml" +- "!include events_monthly_from_collection.yaml" +- "!include events_monthly_from_project.yaml" +- "!include events_monthly_to_artifact.yaml" +- "!include events_monthly_to_collection.yaml" +- "!include events_monthly_to_project.yaml" +- "!include events_weekly_from_artifact.yaml" +- "!include events_weekly_from_collection.yaml" +- "!include events_weekly_from_project.yaml" +- "!include events_weekly_to_artifact.yaml" +- "!include events_weekly_to_collection.yaml" +- "!include events_weekly_to_project.yaml" +- "!include first_contribution_to_project.yaml" +- "!include last_contribution_to_project.yaml" +- "!include onchain_metrics_by_collection.yaml" +- "!include onchain_metrics_by_project.yaml" +- "!include pm_dev_months.yaml" +- "!include pm_new_contribs.yaml" - "!include projects.yaml" - "!include projects_by_collection.yaml" -- "!include code_metrics_by_collection.yaml" -- "!include code_metrics_by_project.yaml" +- "!include repos_by_project.yaml" +- "!include users_monthly_to_project.yaml" diff --git a/apps/hasura/package.json b/apps/hasura/package.json index 003972a9e..b2e85719f 100644 --- a/apps/hasura/package.json +++ b/apps/hasura/package.json @@ -17,7 +17,7 @@ "node": ">=20" }, "scripts": { - "build": "tsc && pnpm metadata:genTables", + "build": "tsc", "lint": "tsc --noEmit && pnpm lint:eslint && pnpm lint:prettier", "lint:eslint": "eslint --ignore-path ../../.gitignore --max-warnings 0 .", "lint:prettier": "prettier --ignore-path ../../.gitignore --log-level warn --check **/*.{js,jsx,ts,tsx,sol,md,json}", @@ -25,7 +25,7 @@ "metadata:pull": "hasura metadata export", "metadata:reload": "hasura metadata reload", "metadata:apply": "hasura metadata apply", - "deploy": "pnpm metadata:reload && pnpm metadata:apply" + "deploy": "pnpm metadata:genTables && pnpm metadata:reload && pnpm metadata:apply" }, "keywords": [], "devDependencies": { diff --git a/apps/hasura/src/genTables.ts b/apps/hasura/src/genTables.ts index 131ab7c47..b5584c9be 100644 --- a/apps/hasura/src/genTables.ts +++ b/apps/hasura/src/genTables.ts @@ -2,16 +2,15 @@ import path from "node:path"; import fs from "node:fs/promises"; import { fileURLToPath } from "node:url"; import * as yaml from "yaml"; +import { exec } from "node:child_process"; +import * as util from "util"; + +const execPromise = util.promisify(exec); const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); // YAML file extension const EXTENSION = ".yaml"; -// Recursively scan this directory for database tables -const modelDir = path.resolve( - __dirname, - "../../../warehouse/dbt/models/marts/", -); // Where to store all table configs const tablesDir = path.resolve( __dirname, @@ -79,14 +78,63 @@ const createConfig = (name: string): TableConfig => ({ ], }); +type ModelConfig = { + name: string; + config: { + meta: { + sync_to_cloudsql: boolean; + }; + }; +}; + async function main(): Promise { - console.log(`Generating tables from ${modelDir}`); + const target = process.env.DBT_TARGET; + if (!target) { + throw new Error("specify a DBT_TARGET"); + } + console.log(`Generating tables from dbt`); + + // FIXME... this isn't very portable + // Run dbt to get the json + const repoRoot = path.resolve("../../"); + const modelsList = await execPromise( + `${repoRoot}/.venv/bin/dbt ls -q --output json --select marts.* --target ${target} --resource-type model`, + { + cwd: repoRoot, + }, + ); + + const modelsConfigRaw = modelsList.stdout.split("\n"); + const modelConfigs: ModelConfig[] = []; + for (const raw of modelsConfigRaw) { + const trimmed = raw.trim(); + if (trimmed === "") { + continue; + } + const modelConfig = JSON.parse(trimmed) as { + name: string; + config: { + meta: { + sync_to_cloudsql: boolean; + }; + }; + }; + modelConfigs.push(modelConfig); + } + const filteredConfigs = modelConfigs.filter((c) => { + return c.config.meta.sync_to_cloudsql === true; + }); + + const tableNames = filteredConfigs.map((c) => { + return c.name; + }); + // Recursively scan all files in the model directory - const allFiles = await fs.readdir(modelDir, { recursive: true }); + //const allFiles = await fs.readdir(modelDir, { recursive: true }); // Get the basename as the table name - const tableNames = allFiles - .filter((f) => f.endsWith(".sql")) - .map((f) => path.basename(f, ".sql")); + // const tableNames = allFiles + // .filter((f) => f.endsWith(".sql")) + // .map((f) => path.basename(f, ".sql")); console.log("Tables:"); console.log(tableNames); // Write the list of tables