feat: any_evm addresses warns if you're tagging contract (#1599)

* This comes with some pretty major refactoring of how we store
  ValidationResults
* Move mustache templating code to separate module
* Create a generic class for storing ValidationResults that will be
  rendered to a GitHub issue comment
* Updates to oss-directory 0.0.15
* Fixes up some minor logging too

ops/external-prs/README.md

@@ -44,7 +44,18 @@ pnpm external-prs oso test-deploy clean {ttl-seconds}
 
 ### OSS-Directory Specific
 
+First configure `.env`.
+
+Then `git clone` to 2 different paths on your filesystem,
+the oss-directory main branch
+and the PR branch to compare.
+
+You can run the app via:
+
 ```bash
 # Handle PR validations
-pnpm external-prs ossd validate-prs {pr} {sha} {main-path} {pr-path}
+pnpm external-prs ossd validate-prs {pr_number} {commit_sha} {main_path} {pr_path}
 ```
+
+If you've configured your GitHub secrets correctly,
+this should post a comment in the PR with the results

ops/external-prs/package.json

@@ -50,7 +50,7 @@
     "lodash": "^4.17.21",
     "mustache": "^4.2.0",
     "octokit": "^3.1.0",
-    "oss-directory": "^0.0.14",
+    "oss-directory": "^0.0.15",
     "tmp-promise": "^3.0.3",
     "ts-dedent": "^2.2.0",
     "winston": "^3.11.0",

ops/external-prs/src/base.ts

@@ -81,7 +81,7 @@ export class GHAppUtils {
     const taggedMessage = `${messageIdText}\n${message}`;
 
     const appId = this.appMeta.appId;
-    console.log(`appId: ${appId}`);
+    logger.info(`Setting status comment with appId: ${appId}`);
     // Search for a comment with the messageId
     // If it doesn't exist, create a comment
     // If it does exist update that comment
@@ -91,11 +91,11 @@ export class GHAppUtils {
       issue_number: pr,
     });
 
-    console.log(allCommentRefs.data.map((c) => c.user));
+    //console.log(allCommentRefs.data.map((c) => c.user));
     const appCommentRefs = allCommentRefs.data.filter((c) => {
       return c.performed_via_github_app?.id === appId;
     });
-    console.log(appCommentRefs);
+    //console.log(appCommentRefs);
 
     // If this app has never commented on this PR, just create it
     if (appCommentRefs.length === 0) {

ops/external-prs/src/ossd/index.ts

@@ -12,14 +12,11 @@ import {
   BlockchainTag,
 } from "oss-directory";
 import duckdb from "duckdb";
-import _ from "lodash";
 import * as util from "util";
 import * as fs from "fs";
-import * as fsPromise from "fs/promises";
 import * as path from "path";
 import * as repl from "repl";
 import columnify from "columnify";
-import mustache from "mustache";
 import { BigQueryOptions } from "@google-cloud/bigquery";
 import {
   EVMNetworkValidator,
@@ -29,13 +26,18 @@ import {
   OptimismValidator,
 } from "@opensource-observer/oss-artifact-validators";
 import { uncheckedCast } from "@opensource-observer/utils";
-import { GithubOutput } from "../github.js";
 import { CheckConclusion, CheckStatus } from "../checks.js";
+import { GithubOutput } from "../github.js";
+import { renderMustacheFromFile } from "./templating.js";
+import { ValidationResults } from "./validation-results.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
-// Should map to the tables field in ./metadata/databases/databases.yaml
+function relativeDir(...args: string[]) {
+  return path.join(__dirname, ...args);
+}
 
+// Should map to the tables field in ./metadata/databases/databases.yaml
 function jsonlExport<T>(path: string, arr: Array<T>): Promise<void> {
   return new Promise((resolve, reject) => {
     const stream = fs.createWriteStream(path, "utf-8");
@@ -206,10 +208,6 @@ interface RpcUrlArgs {
 
 type ValidatePRArgs = OSSDirectoryPullRequestArgs & RpcUrlArgs;
 
-function relativeDir(...args: string[]) {
-  return path.join(__dirname, ...args);
-}
-
 async function runParameterizedQuery(
   db: duckdb.Database,
   name: string,
@@ -226,14 +224,6 @@ async function runParameterizedQuery(
   return dbAll(query);
 }
 
-async function renderMustacheFromFile(
-  filePath: string,
-  params?: Record<string, unknown>,
-) {
-  const raw = await fsPromise.readFile(filePath, "utf-8");
-  return mustache.render(raw, params);
-}
-
 // | Projects             | {{ projects.existing }} {{ projects.added }} | {{projects.removed}} | {{ projects.updated }}
 type ProjectSummary = {
   project_slug: string;
@@ -631,30 +621,7 @@ class OSSDirectoryPullRequest {
     });
     await this.loadValidators(urls);
 
-    // Embedded data structure for storing validation results
-    type ValidationItem = {
-      name: string;
-      messages: string[];
-      errors: string[];
-      warnings: string[];
-      successes: string[];
-    };
-    const results: Record<string, ValidationItem> = {};
-    // Add a name to the results
-    const ensureNameInResult = (name: string) => {
-      const item = results[name];
-      if (!item) {
-        results[name] = {
-          name,
-          messages: [],
-          errors: [],
-          warnings: [],
-          successes: [],
-        };
-      }
-      return results[name];
-    };
-
+    const results = await ValidationResults.create();
     const addressesToValidate =
       this.changes.artifacts.toValidate.blockchain.map((b) => b.address);
     logger.info({
@@ -669,29 +636,14 @@ class OSSDirectoryPullRequest {
         const validator =
           this.validators[uncheckedCast<BlockchainNetwork>(network)];
         if (!validator) {
-          logger.warn({
-            message: `no validator found for network ${network}`,
-            network: network,
-          });
-          ensureNameInResult(address).warnings.push(
+          results.addWarning(
             `no automated validators exist on ${network} to check tags=[${item.tags}]. Please check manually.`,
+            address,
+            { network },
           );
           //throw new Error(`No validator found for network "${network}"`);
           continue;
         }
-        const createValidatorMapping = (
-          validator: EVMNetworkValidator,
-        ): Partial<
-          Record<BlockchainTag, (name: string) => Promise<boolean>>
-        > => {
-          const mapping = {
-            eoa: (addr: string) => validator.isEOA(addr),
-            contract: (addr: string) => validator.isContract(addr),
-            deployer: (addr: string) => validator.isDeployer(addr),
-          };
-          return mapping;
-        };
-        const validatorMappings = createValidatorMapping(validator);
 
         logger.info({
           message: `validating address ${address} on ${network} for [${item.tags}]`,
@@ -702,54 +654,50 @@ class OSSDirectoryPullRequest {
 
         for (const rawTag of item.tags) {
           const tag = uncheckedCast<BlockchainTag>(rawTag);
-          const validatorFn = validatorMappings[tag];
-          if (!validatorFn) {
-            logger.error({
-              message: `ERROR: missing validator for ${tag} on network=${network}`,
-              tag,
-              network,
-            });
-            ensureNameInResult(address).warnings.push(
-              `missing validator for ${tag} on network=${network}`,
-            );
-          } else if (!(await validatorFn(address))) {
-            ensureNameInResult(address).errors.push(
-              `is not a '${tag}' on ${network}`,
-            );
+          const genericChecker = async (fn: () => Promise<boolean>) => {
+            if (!(await fn())) {
+              results.addError(
+                `${address} is not a ${tag} on ${network}`,
+                address,
+                { address, tag, network },
+              );
+            } else {
+              results.addSuccess(
+                `${address} is a '${tag}' on ${network}`,
+                address,
+                { address, tag, network },
+              );
+            }
+          };
+          if (tag === "eoa") {
+            await genericChecker(() => validator.isEOA(address));
+          } else if (tag === "contract") {
+            if (network === "any_evm") {
+              results.addWarning(
+                `addresses with the 'contract' tag should enumerate all networks that it is deployed on, rather than use 'any_evm'`,
+                address,
+                { address, tag, network },
+              );
+            } else {
+              await genericChecker(() => validator.isContract(address));
+            }
+          } else if (tag === "deployer") {
+            await genericChecker(() => validator.isDeployer(address));
           } else {
-            ensureNameInResult(address).successes.push(
-              `is a '${tag}' on ${network}`,
+            results.addWarning(
+              `missing validator for ${tag} on ${network}`,
+              address,
+              { tag, network },
             );
           }
         }
       }
     }
 
-    // Summarize results
-    const items: ValidationItem[] = _.values(results);
-    const numErrors = _.sumBy(
-      items,
-      (item: ValidationItem) => item.errors.length,
-    );
-    const numWarningsMessages = _.sumBy(
-      items,
-      (item: ValidationItem) => item.warnings.length + item.messages.length,
+    // Render the results to GitHub PR
+    const { numErrors, summaryMessage, commentBody } = await results.render(
+      args.sha,
     );
-    const summaryMessage =
-      numErrors > 0
-        ? `⛔ Found ${numErrors} errors ⛔`
-        : numWarningsMessages > 0
-          ? "⚠️ Please review messages before approving ⚠️"
-          : "✅ Good to go as long as status checks pass";
-    const commentBody = await renderMustacheFromFile(
-      relativeDir("messages", "validation-message.md"),
-      {
-        sha: args.sha,
-        summaryMessage,
-        validationItems: items,
-      },
-    );
-
     // Update the PR comment
     await args.appUtils.setStatusComment(args.pr, commentBody);
     // Update the PR status

ops/external-prs/src/ossd/templating.ts

@@ -0,0 +1,12 @@
+import * as fsPromise from "fs/promises";
+import mustache from "mustache";
+
+async function renderMustacheFromFile(
+  filePath: string,
+  params?: Record<string, unknown>,
+) {
+  const raw = await fsPromise.readFile(filePath, "utf-8");
+  return mustache.render(raw, params);
+}
+
+export { renderMustacheFromFile };