feat: Hasura3 scripts (#1860)

* Move hasura2 out of the pnpm workspace
* Move hasura3 into the pnpm workspace
* Add typescript code in hasura3 to script changes to metadata
* This will make all models publicly accessible

.github/workflows/deploy-hasura.yml

@@ -15,7 +15,7 @@ on:
     branches:
       - main
     paths:
-      - apps/hasura/**
+      - apps/hasura2/**
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

apps/hasura/.eslintrc.json → apps/hasura2/.eslintrc.json

@@ -2,6 +2,6 @@
   "extends": ["../../.eslintrc.js"],
   "root": false,
   "parserOptions": {
-    "project": ["./apps/hasura/tsconfig.json"]
+    "project": ["./apps/hasura2/tsconfig.json"]
   }
 }

apps/hasura/package.json → apps/hasura2/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@opensource-observer/hasura",
+  "name": "@opensource-observer/hasura2",
   "version": "0.0.1",
   "description": "API service for OSO",
   "author": "Kariba Labs",

apps/hasura3/.env.example

@@ -0,0 +1,4 @@
+
+# Hasura personal access token
+# `ddn auth print-pat`
+HASURA_DDN_PAT=

apps/hasura3/.eslintrc.json

@@ -0,0 +1,7 @@
+{
+  "extends": ["../../.eslintrc.js"],
+  "root": false,
+  "parserOptions": {
+    "project": ["./apps/hasura3/tsconfig.json"]
+  }
+}

apps/hasura3/README.md

@@ -0,0 +1,51 @@
+# Hasura configuration
+
+This directory stores all configurations for the Hasura deployment.
+Hasura is currently setup with only a single subgraph, `oso_subgraph`,
+with a single connector `oso_clickhouse`.
+
+_Note: This only works for Hasura version 3 (aka DDN)._
+
+## Setup
+
+Copy `.env.example` to `.env` and set the environment variables as needed.
+You can get your Hasura PAT by running
+
+```bash
+ddn auth print-pat
+```
+
+In `./oso_subgraph/`, copy `.env.oso_subgraph.example` to `.env.oso_subgraph.cloud` and `.env.oso_subgraph.local`. These need to be configured to your Clickhouse connector deployment. The local deployment will be launched by Docker compose, where was the cloud deployment is hosted by Hasura.
+
+In `./oso_subgraph/connector/oso_clickhouse`, copy `.env.example` to `.env.cloud` and `.env.local`. Populate with Clickhouse credentials.
+
+## Sync schema with Clickhouse
+
+## Configure
+
+You can modify any files to update the Hasura configuration.
+See the
+[Hasura docs](https://hasura.io/docs/3.0/) to learn more.
+
+Note that anything in `./oso_subgraph/metadata/` will be overwritten by the build step.
+
+## Build
+
+This will idempotently update table configurations for all tables found in the Clickhouse database.
+This needs to be run every time the schema changes.
+For more information, see `./src/cli.ts`.
+
+```bash
+pnpm build
+```
+
+## Deploy
+
+To reload the database schemas and apply the metadata configurations, run:
+
+```bash
+pnpm run deploy
+```
+
+For more information on how to manage metadata, see the
+[Hasura guide](https://hasura.io/docs/latest/migrations-metadata-seeds/manage-metadata/#reload-metadata).

apps/hasura3/globals/auth-config.cloud.hml

@@ -4,5 +4,5 @@ definition:
   allowRoleEmulationBy: admin
   mode:
     webhook:
-      url: http://auth-hook.default:8080/webhook/ddn?role=admin
-      method: Post
+      url: https://www.opensource.observer/api/v1/auth
+      method: GET

apps/hasura3/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@opensource-observer/hasura3",
+  "version": "0.0.1",
+  "description": "API service for OSO",
+  "author": "Kariba Labs",
+  "license": "Apache-2.0",
+  "private": true,
+  "bin": "./dist/src/cli.js",
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/opensource-observer/oso.git"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build:ts": "tsc",
+    "build": "pnpm build:ts && pnpm metadata:update",
+    "lint": "tsc --noEmit && pnpm lint:eslint && pnpm lint:prettier",
+    "lint:eslint": "eslint --ignore-path ../../.gitignore --max-warnings 0 .",
+    "lint:prettier": "prettier --ignore-path ../../.gitignore --log-level warn --check **/*.{js,jsx,ts,tsx,sol,md,json}",
+    "metadata:update": "dotenv -- node --loader ts-node/esm src/cli.ts",
+    "metadata:introspect:db": "ddn connector introspect --connector oso_subgraph/connector/oso_clickhouse/connector.local.yaml",
+    "metadata:sync:db": "ddn connector-link update oso_clickhouse --subgraph oso_subgraph/subgraph.yaml --env-file oso_subgraph/.env.oso_subgraph.local --add-all-resources",
+    "metadata:build:local": "ddn supergraph build local --output-dir engine --subgraph-env-file oso_subgraph:oso_subgraph/.env.oso_subgraph.local",
+    "metadata:deploy:cloud": "ddn supergraph build create --supergraph supergraph.cloud.yaml --subgraph-env-file oso_subgraph:oso_subgraph/.env.oso_subgraph.cloud",
+    "start": "dotenv -- docker compose up --build --watch",
+    "deploy": "pnpm metadata:update"
+  },
+  "keywords": [],
+  "devDependencies": {
+    "@types/node": "^20.14.12",
+    "dotenv-cli": "^7.4.2",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.4"
+  },
+  "dependencies": {
+    "glob": "^11.0.0",
+    "yaml": "^2.5.0"
+  }
+}

apps/hasura3/src/cli.ts

@@ -0,0 +1,151 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { glob } from "glob";
+import * as yaml from "yaml";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+// YAML file extension
+const EXTENSION = ".hml";
+const DOCUMENT_PREFIX = "---\n";
+const DOCUMENT_POSTFIX = "";
+const DOCUMENT_DELIMITER = "\n";
+
+// Where to store all table configs
+const metadataDir = path.resolve(__dirname, "../oso_subgraph/metadata/");
+// Should map to the tables field in ./metadata/databases/databases.yaml
+const metadataGlob = path.format({
+  dir: metadataDir,
+  name: "*",
+  ext: EXTENSION,
+});
+
+// Schema for Hasura table configuration
+type ModelPermissions = {
+  kind: "ModelPermissions";
+  version: string;
+  definition: {
+    modelName: string;
+    permissions: {
+      role: string;
+      select: {
+        filter: null;
+      };
+    }[];
+  };
+};
+
+type TypePermissions = {
+  kind: "TypePermissions";
+  version: string;
+  definition: {
+    typeName: string;
+    permissions: {
+      role: string;
+      output: {
+        allowedFields: string[];
+      };
+    }[];
+  };
+};
+
+const ensureModelPermissions = (obj: Partial<ModelPermissions>) => {
+  if (!obj?.definition?.permissions) {
+    throw new Error("Malformed ModelPermissions");
+  }
+  // Skip if already has anonymous permission
+  const hasAnon = obj.definition.permissions.find(
+    (p) => p.role === "anonymous",
+  );
+  if (hasAnon) {
+    return { ...obj };
+  }
+
+  // Add anonymous permission
+  return {
+    ...obj,
+    definition: {
+      ...obj.definition,
+      permissions: [
+        ...obj.definition.permissions,
+        {
+          role: "anonymous",
+          select: { filter: null },
+        },
+      ],
+    },
+  };
+};
+
+const ensureTypePermissions = (obj: Partial<TypePermissions>) => {
+  if (!obj?.definition?.permissions) {
+    throw new Error("Malformed TypePermissions");
+  }
+
+  // We need the admin permission to copy from
+  const adminPerm = obj.definition.permissions.find((p) => p.role === "admin");
+  if (!adminPerm) {
+    throw new Error("TypePermissions must have pre-existing admin permission");
+  }
+
+  // Exclude any pre-existing anonymous permissions
+  const permsWithoutAnon = obj.definition.permissions.filter(
+    (p) => p.role !== "anonymous",
+  );
+
+  // Copy admin permission to anonymous permissions
+  return {
+    ...obj,
+    definition: {
+      ...obj.definition,
+      permissions: [
+        ...permsWithoutAnon,
+        {
+          role: "anonymous",
+          output: {
+            allowedFields: [...adminPerm.output.allowedFields],
+          },
+        },
+      ],
+    },
+  };
+};
+
+const ensureDocument = (obj: any) => {
+  if (obj?.kind === "ModelPermissions") {
+    return ensureModelPermissions(obj);
+  } else if (obj?.kind === "TypePermissions") {
+    return ensureTypePermissions(obj);
+  } else {
+    return obj;
+  }
+};
+
+async function main(): Promise<void> {
+  // Scan all metadata files
+  //const allFiles = await fs.readdir(metadataDir, { recursive: false });
+  const allFiles = await glob(metadataGlob);
+  console.log(allFiles);
+  for (const file of allFiles) {
+    console.log(`Updating ${file}...`);
+    const strContents = await fs.readFile(file);
+    const docs = yaml.parseAllDocuments(strContents.toString());
+    const newDocs = docs.map((doc) => {
+      const docObj = doc.toJSON();
+      return ensureDocument(docObj);
+    });
+    const newStrings = newDocs.map(
+      (d) => DOCUMENT_PREFIX + yaml.stringify(d) + DOCUMENT_POSTFIX,
+    );
+    const newFile = newStrings.join(DOCUMENT_DELIMITER);
+    await fs.writeFile(file, newFile);
+  }
+}
+
+main()
+  .then(() => console.log("Done"))
+  .catch((e) => {
+    //console.warn(e);
+    throw e;
+  });

apps/hasura3/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "outDir": "dist",
+    "target": "ES6",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "declaration": true,
+    "declarationMap": true,
+    "esModuleInterop": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "noImplicitAny": true,
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "strictNullChecks": true,
+    "strictPropertyInitialization": false
+  },
+  "exclude": ["node_modules"],
+  "include": ["./src/*.ts", "./src/**/*.ts", "./test"]
+}

package.json

@@ -15,13 +15,13 @@
     "build:dbt:docs": "poetry run dbt docs generate",
     "build:docs": "turbo run build --filter=@opensource-observer/docs",
     "build:frontend": "turbo run build --filter=@opensource-observer/frontend",
-    "build:hasura": "turbo run build --filter=@opensource-observer/hasura",
+    "build:hasura": "turbo run build --filter=@opensource-observer/hasura2",
     "build:ops": "turbo run build --filter=@opensource-observer/ops-*",
     "copy": "yarn copy:frontend && yarn copy:docs && yarn copy:html",
     "copy:docs": "mkdir -p ./build/docs/ && cp -r ./apps/docs/build/* ./build/docs/",
     "copy:frontend": "mkdir -p ./build/ && cp -r ./apps/frontend/out/* ./build/ && cp ./apps/frontend/_redirects ./build/",
     "copy:html": "find build/ -name '*.html' -type f | grep -v index.html | sed s/\\.html$// | xargs -I _ bash -c 'mkdir -p _ && cp -v _.html _/index.html'",
-    "deploy:hasura": "turbo run deploy --filter=@opensource-observer/hasura --parallel",
+    "deploy:hasura": "turbo run deploy --filter=@opensource-observer/hasura2 --parallel",
     "deploy:site": "turbo run build --filter=@opensource-observer/docs --filter=@opensource-observer/frontend && turbo run deploy --filter=@opensource-observer/frontend && yarn copy",
     "dev:docs": "turbo run dev --filter=@opensource-observer/docs --parallel",
     "dev:frontend": "turbo run dev --filter=@opensource-observer/frontend --parallel",