From cf3a4514798cdc6211665bd22b6d9488954ef9fd Mon Sep 17 00:00:00 2001 From: Raymond Cheng Date: Sun, 28 Jul 2024 14:24:02 -0700 Subject: [PATCH] feat: Hasura configurations for public access (#1865) * This will run metadata sync and update to allow public access * Updated the README for instructions on how to run as well --- apps/hasura3/README.md | 55 +++++++++++++++---- apps/hasura3/globals/auth-config.cloud.hml | 2 +- .../metadata/ArtifactsByProjectV1.hml | 16 +++++- .../oso_subgraph/metadata/ArtifactsV1.hml | 13 ++++- .../metadata/CodeMetricsByProjectV1.hml | 28 +++++++++- .../oso_subgraph/metadata/CollectionsV1.hml | 13 ++++- .../oso_subgraph/metadata/ContractsV0.hml | 10 +++- .../EventIndexingStatusByProjectV1.hml | 16 +++++- .../oso_subgraph/metadata/EventTypesV1.hml | 8 ++- .../metadata/EventsDailyToArtifact.hml | 11 +++- .../metadata/EventsDailyToCollection.hml | 11 +++- .../metadata/EventsDailyToProject.hml | 11 +++- .../metadata/EventsMonthlyToArtifact.hml | 11 +++- .../metadata/EventsMonthlyToCollection.hml | 11 +++- .../metadata/EventsMonthlyToProject.hml | 11 +++- .../metadata/EventsWeeklyToArtifact.hml | 11 +++- .../metadata/EventsWeeklyToCollection.hml | 11 +++- .../metadata/EventsWeeklyToProject.hml | 11 +++- .../metadata/OnchainMetricsByProjectV1.hml | 27 ++++++++- .../metadata/ProjectsByCollectionV1.hml | 15 ++++- .../oso_subgraph/metadata/ProjectsV1.hml | 13 ++++- .../hasura3/oso_subgraph/metadata/UsersV1.hml | 14 ++++- .../metadata/oso_clickhouse-types.hml | 1 - .../oso_subgraph/metadata/oso_clickhouse.hml | 1 + apps/hasura3/package.json | 5 +- 25 files changed, 300 insertions(+), 36 deletions(-) diff --git a/apps/hasura3/README.md b/apps/hasura3/README.md index 441b6901f..00fd289d7 100644 --- a/apps/hasura3/README.md +++ b/apps/hasura3/README.md @@ -8,6 +8,16 @@ _Note: This only works for Hasura version 3 (aka DDN)._ ## Setup +### Install `ddn` + +First install the `ddn` CLI tool from Hasura + +```bash +curl -L https://graphql-engine-cdn.hasura.io/ddn/cli/v3/get.sh | bash +``` + +### Configure environment + Copy `.env.example` to `.env` and set the environment variables as needed. You can get your Hasura PAT by running @@ -19,33 +29,54 @@ In `./oso_subgraph/`, copy `.env.oso_subgraph.example` to `.env.oso_subgraph.clo In `./oso_subgraph/connector/oso_clickhouse`, copy `.env.example` to `.env.cloud` and `.env.local`. Populate with Clickhouse credentials. +### Build + +```bash +pnpm install +pnpm build +``` + ## Sync schema with Clickhouse -## Configure +First, start a local Hasura instance. This is required to run the Clickhouse connector. + +```bash +pnpm start +``` + +In a separate terminal, pull the latest Clickhouse schema + +```bash +pnpm sync +``` -You can modify any files to update the Hasura configuration. +## Configure metadata + +From here, you can modify any files to update the Hasura configuration. See the [Hasura docs](https://hasura.io/docs/3.0/) to learn more. -Note that anything in `./oso_subgraph/metadata/` will be overwritten by the build step. - -## Build +We include a script for automatically modifying the metadata +in `./src/cli.ts`. This will currently just make all models +publicly available. -This will idempotently update table configurations for all tables found in the Clickhouse database. -This needs to be run every time the schema changes. -For more information, see `./src/cli.ts`. +To run this script ```bash -pnpm build +pnpm metadata:update ``` ## Deploy -To reload the database schemas and apply the metadata configurations, run: +To apply the metadata configurations to Hasura cloud, run: ```bash pnpm run deploy ``` -For more information on how to manage metadata, see the -[Hasura guide](https://hasura.io/docs/latest/migrations-metadata-seeds/manage-metadata/#reload-metadata). +This will output a build ID, which you'll need to apply as default: + +```bash +ddn supergraph build apply BUILD_ID +# ddn supergraph build apply 030c5f3ce0 +``` diff --git a/apps/hasura3/globals/auth-config.cloud.hml b/apps/hasura3/globals/auth-config.cloud.hml index ad5626951..db7766581 100644 --- a/apps/hasura3/globals/auth-config.cloud.hml +++ b/apps/hasura3/globals/auth-config.cloud.hml @@ -5,4 +5,4 @@ definition: mode: webhook: url: https://www.opensource.observer/api/v1/auth - method: GET + method: Get diff --git a/apps/hasura3/oso_subgraph/metadata/ArtifactsByProjectV1.hml b/apps/hasura3/oso_subgraph/metadata/ArtifactsByProjectV1.hml index 3c43f5391..35fe9c8ff 100644 --- a/apps/hasura3/oso_subgraph/metadata/ArtifactsByProjectV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/ArtifactsByProjectV1.hml @@ -75,6 +75,18 @@ definition: - projectName - projectNamespace - projectSource + - role: anonymous + output: + allowedFields: + - artifactId + - artifactName + - artifactNamespace + - artifactSource + - artifactSourceId + - projectId + - projectName + - projectNamespace + - projectSource --- kind: ObjectBooleanExpressionType @@ -168,4 +180,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/ArtifactsV1.hml b/apps/hasura3/oso_subgraph/metadata/ArtifactsV1.hml index ac4a85297..047adc515 100644 --- a/apps/hasura3/oso_subgraph/metadata/ArtifactsV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/ArtifactsV1.hml @@ -57,6 +57,15 @@ definition: - artifactSource - artifactSourceId - artifactUrl + - role: anonymous + output: + allowedFields: + - artifactId + - artifactName + - artifactNamespace + - artifactSource + - artifactSourceId + - artifactUrl --- kind: ObjectBooleanExpressionType @@ -132,4 +141,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/CodeMetricsByProjectV1.hml b/apps/hasura3/oso_subgraph/metadata/CodeMetricsByProjectV1.hml index de39282f8..c15100d29 100644 --- a/apps/hasura3/oso_subgraph/metadata/CodeMetricsByProjectV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/CodeMetricsByProjectV1.hml @@ -147,6 +147,30 @@ definition: - projectSource - repositoryCount - starCount + - role: anonymous + output: + allowedFields: + - activeDeveloperCount6Months + - closedIssueCount6Months + - commitCount6Months + - contributorCount + - contributorCount6Months + - displayName + - eventSource + - firstCommitDate + - forkCount + - fulltimeDeveloperAverage6Months + - lastCommitDate + - mergedPullRequestCount6Months + - newContributorCount6Months + - openedIssueCount6Months + - openedPullRequestCount6Months + - projectId + - projectName + - projectNamespace + - projectSource + - repositoryCount + - starCount --- kind: ObjectBooleanExpressionType @@ -312,4 +336,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/CollectionsV1.hml b/apps/hasura3/oso_subgraph/metadata/CollectionsV1.hml index 2dd4590dc..6ef5bd317 100644 --- a/apps/hasura3/oso_subgraph/metadata/CollectionsV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/CollectionsV1.hml @@ -57,6 +57,15 @@ definition: - collectionSource - description - displayName + - role: anonymous + output: + allowedFields: + - collectionId + - collectionName + - collectionNamespace + - collectionSource + - description + - displayName --- kind: ObjectBooleanExpressionType @@ -132,4 +141,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/ContractsV0.hml b/apps/hasura3/oso_subgraph/metadata/ContractsV0.hml index a47216722..56c8250d6 100644 --- a/apps/hasura3/oso_subgraph/metadata/ContractsV0.hml +++ b/apps/hasura3/oso_subgraph/metadata/ContractsV0.hml @@ -39,6 +39,12 @@ definition: - artifactSource - contractAddress - rootDeployerAddress + - role: anonymous + output: + allowedFields: + - artifactSource + - contractAddress + - rootDeployerAddress --- kind: ObjectBooleanExpressionType @@ -96,4 +102,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventIndexingStatusByProjectV1.hml b/apps/hasura3/oso_subgraph/metadata/EventIndexingStatusByProjectV1.hml index 5df4b20dd..90879f1b2 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventIndexingStatusByProjectV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventIndexingStatusByProjectV1.hml @@ -75,6 +75,18 @@ definition: - projectName - projectNamespace - projectSource + - role: anonymous + output: + allowedFields: + - eventSource + - eventType + - eventfulDayCount + - firstEventDate + - lastEventDate + - projectId + - projectName + - projectNamespace + - projectSource --- kind: ObjectBooleanExpressionType @@ -168,4 +180,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventTypesV1.hml b/apps/hasura3/oso_subgraph/metadata/EventTypesV1.hml index bb64cf668..6953ba6c7 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventTypesV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventTypesV1.hml @@ -27,6 +27,10 @@ definition: output: allowedFields: - eventType + - role: anonymous + output: + allowedFields: + - eventType --- kind: ObjectBooleanExpressionType @@ -72,4 +76,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsDailyToArtifact.hml b/apps/hasura3/oso_subgraph/metadata/EventsDailyToArtifact.hml index bbf4bd102..43a1ad773 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsDailyToArtifact.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsDailyToArtifact.hml @@ -45,6 +45,13 @@ definition: - artifactId - bucketDay - eventType + - role: anonymous + output: + allowedFields: + - amount + - artifactId + - bucketDay + - eventType --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsDailyToCollection.hml b/apps/hasura3/oso_subgraph/metadata/EventsDailyToCollection.hml index 34358e42e..a3eee3ddf 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsDailyToCollection.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsDailyToCollection.hml @@ -45,6 +45,13 @@ definition: - bucketDay - collectionId - eventType + - role: anonymous + output: + allowedFields: + - amount + - bucketDay + - collectionId + - eventType --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsDailyToProject.hml b/apps/hasura3/oso_subgraph/metadata/EventsDailyToProject.hml index a5854b053..4b5e01fd0 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsDailyToProject.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsDailyToProject.hml @@ -45,6 +45,13 @@ definition: - bucketDay - eventType - projectId + - role: anonymous + output: + allowedFields: + - amount + - bucketDay + - eventType + - projectId --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToArtifact.hml b/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToArtifact.hml index a531d32b3..54c9fb851 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToArtifact.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToArtifact.hml @@ -45,6 +45,13 @@ definition: - artifactId - bucketMonth - eventType + - role: anonymous + output: + allowedFields: + - amount + - artifactId + - bucketMonth + - eventType --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToCollection.hml b/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToCollection.hml index eab339583..e4b4300b9 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToCollection.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToCollection.hml @@ -45,6 +45,13 @@ definition: - bucketMonth - collectionId - eventType + - role: anonymous + output: + allowedFields: + - amount + - bucketMonth + - collectionId + - eventType --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToProject.hml b/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToProject.hml index 5d4fe50cb..5d878d291 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToProject.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsMonthlyToProject.hml @@ -45,6 +45,13 @@ definition: - bucketMonth - eventType - projectId + - role: anonymous + output: + allowedFields: + - amount + - bucketMonth + - eventType + - projectId --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToArtifact.hml b/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToArtifact.hml index a58fd1fb9..61a117a99 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToArtifact.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToArtifact.hml @@ -45,6 +45,13 @@ definition: - artifactId - bucketWeek - eventType + - role: anonymous + output: + allowedFields: + - amount + - artifactId + - bucketWeek + - eventType --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToCollection.hml b/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToCollection.hml index feec96894..07e11bc0a 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToCollection.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToCollection.hml @@ -45,6 +45,13 @@ definition: - bucketWeek - collectionId - eventType + - role: anonymous + output: + allowedFields: + - amount + - bucketWeek + - collectionId + - eventType --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToProject.hml b/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToProject.hml index cfd17f8e5..c64b05be0 100644 --- a/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToProject.hml +++ b/apps/hasura3/oso_subgraph/metadata/EventsWeeklyToProject.hml @@ -45,6 +45,13 @@ definition: - bucketWeek - eventType - projectId + - role: anonymous + output: + allowedFields: + - amount + - bucketWeek + - eventType + - projectId --- kind: ObjectBooleanExpressionType @@ -108,4 +115,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/OnchainMetricsByProjectV1.hml b/apps/hasura3/oso_subgraph/metadata/OnchainMetricsByProjectV1.hml index d27498bc0..79039ca10 100644 --- a/apps/hasura3/oso_subgraph/metadata/OnchainMetricsByProjectV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/OnchainMetricsByProjectV1.hml @@ -141,6 +141,29 @@ definition: - returningAddressCount90Days - transactionCount - transactionCount6Months + - role: anonymous + output: + allowedFields: + - activeContractCount90Days + - addressCount + - addressCount90Days + - daysSinceFirstTransaction + - displayName + - eventSource + - gasFeesSum + - gasFeesSum6Months + - highActivityAddressCount90Days + - lowActivityAddressCount90Days + - mediumActivityAddressCount90Days + - multiProjectAddressCount90Days + - newAddressCount90Days + - projectId + - projectName + - projectNamespace + - projectSource + - returningAddressCount90Days + - transactionCount + - transactionCount6Months --- kind: ObjectBooleanExpressionType @@ -300,4 +323,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/ProjectsByCollectionV1.hml b/apps/hasura3/oso_subgraph/metadata/ProjectsByCollectionV1.hml index 4ac0f8626..b9910aafe 100644 --- a/apps/hasura3/oso_subgraph/metadata/ProjectsByCollectionV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/ProjectsByCollectionV1.hml @@ -69,6 +69,17 @@ definition: - projectName - projectNamespace - projectSource + - role: anonymous + output: + allowedFields: + - collectionId + - collectionName + - collectionNamespace + - collectionSource + - projectId + - projectName + - projectNamespace + - projectSource --- kind: ObjectBooleanExpressionType @@ -156,4 +167,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/ProjectsV1.hml b/apps/hasura3/oso_subgraph/metadata/ProjectsV1.hml index 827f68d96..405c5e191 100644 --- a/apps/hasura3/oso_subgraph/metadata/ProjectsV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/ProjectsV1.hml @@ -57,6 +57,15 @@ definition: - projectName - projectNamespace - projectSource + - role: anonymous + output: + allowedFields: + - description + - displayName + - projectId + - projectName + - projectNamespace + - projectSource --- kind: ObjectBooleanExpressionType @@ -132,4 +141,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/UsersV1.hml b/apps/hasura3/oso_subgraph/metadata/UsersV1.hml index cd5005cf8..7e24ac744 100644 --- a/apps/hasura3/oso_subgraph/metadata/UsersV1.hml +++ b/apps/hasura3/oso_subgraph/metadata/UsersV1.hml @@ -63,6 +63,16 @@ definition: - userId - userSource - userSourceId + - role: anonymous + output: + allowedFields: + - bio + - displayName + - profilePictureUrl + - url + - userId + - userSource + - userSourceId --- kind: ObjectBooleanExpressionType @@ -144,4 +154,6 @@ definition: - role: admin select: filter: null - + - role: anonymous + select: + filter: null diff --git a/apps/hasura3/oso_subgraph/metadata/oso_clickhouse-types.hml b/apps/hasura3/oso_subgraph/metadata/oso_clickhouse-types.hml index 7c6da2fa2..fa69f5350 100644 --- a/apps/hasura3/oso_subgraph/metadata/oso_clickhouse-types.hml +++ b/apps/hasura3/oso_subgraph/metadata/oso_clickhouse-types.hml @@ -61,4 +61,3 @@ definition: representation: Int64 graphql: comparisonExpressionTypeName: Oso_Int64ComparisonExp - diff --git a/apps/hasura3/oso_subgraph/metadata/oso_clickhouse.hml b/apps/hasura3/oso_subgraph/metadata/oso_clickhouse.hml index c6e89cdfb..ab7673f2c 100644 --- a/apps/hasura3/oso_subgraph/metadata/oso_clickhouse.hml +++ b/apps/hasura3/oso_subgraph/metadata/oso_clickhouse.hml @@ -1,3 +1,4 @@ +--- kind: DataConnectorLink version: v1 definition: diff --git a/apps/hasura3/package.json b/apps/hasura3/package.json index 4eba91d1b..0b79e1e2b 100644 --- a/apps/hasura3/package.json +++ b/apps/hasura3/package.json @@ -18,7 +18,7 @@ }, "scripts": { "build:ts": "tsc", - "build": "pnpm build:ts && pnpm metadata:update", + "build": "pnpm build:ts", "lint": "tsc --noEmit && pnpm lint:eslint && pnpm lint:prettier", "lint:eslint": "eslint --ignore-path ../../.gitignore --max-warnings 0 .", "lint:prettier": "prettier --ignore-path ../../.gitignore --log-level warn --check **/*.{js,jsx,ts,tsx,sol,md,json}", @@ -28,7 +28,8 @@ "metadata:build:local": "ddn supergraph build local --output-dir engine --subgraph-env-file oso_subgraph:oso_subgraph/.env.oso_subgraph.local", "metadata:deploy:cloud": "ddn supergraph build create --supergraph supergraph.cloud.yaml --subgraph-env-file oso_subgraph:oso_subgraph/.env.oso_subgraph.cloud", "start": "dotenv -- docker compose up --build --watch", - "deploy": "pnpm metadata:update" + "sync": "pnpm metadata:introspect:db && pnpm metadata:sync:db ", + "deploy": "pnpm metadata:deploy:cloud" }, "keywords": [], "devDependencies": {