Stub Out ETL Endpoint

packages/local_db/seed.sample.json

@@ -175,5 +175,103 @@
         }
       }
     }
+  },
+  "etl-process-log": {
+    "20240101": {
+      "status": "success",
+      "message": "string"
+    }
+  },
+  "raw-oso-response": {
+    "20240101": [
+      {
+        "active_contract_count_90_days": 1,
+        "address_count": 2,
+        "address_count_90_days": 2,
+        "days_since_first_transaction": 201,
+        "display_name": "Daimo",
+        "event_source": "OPTIMISM",
+        "gas_fees_sum": 0.00082130956988073,
+        "gas_fees_sum_6_months": 0.0008183154836908259,
+        "high_activity_address_count_90_days": 0,
+        "low_activity_address_count_90_days": 1,
+        "medium_activity_address_count_90_days": 1,
+        "multi_project_address_count_90_days": 0,
+        "new_address_count_90_days": 0,
+        "project_id": "6RbJ7fhcEoin39N2pN8WMvaRQe3TVP_JaohQ7nvojCs=",
+        "project_name": "daimo-eth",
+        "project_namespace": "oso",
+        "project_source": "OSS_DIRECTORY",
+        "returning_address_count_90_days": 2,
+        "transaction_count": 34,
+        "transaction_count_6_months": 30
+      },
+      {
+        "active_contract_count_90_days": 1,
+        "address_count": 2,
+        "address_count_90_days": 2,
+        "days_since_first_transaction": 242,
+        "display_name": "Bankless-Africa",
+        "event_source": "OPTIMISM",
+        "gas_fees_sum": 0.0005384550051846021,
+        "gas_fees_sum_6_months": 0.000526745507020252,
+        "high_activity_address_count_90_days": 0,
+        "low_activity_address_count_90_days": 1,
+        "medium_activity_address_count_90_days": 1,
+        "multi_project_address_count_90_days": 2,
+        "new_address_count_90_days": 0,
+        "project_id": "TnnHcvj2jZDYBPeLzwAMUzlynJygKL7dJnP2imJzQYk=",
+        "project_name": "bankless-africa",
+        "project_namespace": "oso",
+        "project_source": "OSS_DIRECTORY",
+        "returning_address_count_90_days": 2,
+        "transaction_count": 62,
+        "transaction_count_6_months": 54
+      },
+      {
+        "active_contract_count_90_days": 1,
+        "address_count": 3,
+        "address_count_90_days": 2,
+        "days_since_first_transaction": 200,
+        "display_name": "Endangered Tokens: Endangered Trees as ReFi Biodiversity Assets ",
+        "event_source": "OPTIMISM",
+        "gas_fees_sum": 0.000031688086604034,
+        "gas_fees_sum_6_months": 0.000021458559807634,
+        "high_activity_address_count_90_days": 0,
+        "low_activity_address_count_90_days": 2,
+        "medium_activity_address_count_90_days": 0,
+        "multi_project_address_count_90_days": 0,
+        "new_address_count_90_days": 1,
+        "project_id": "NEpTyRk2iJgqJmeLr4-u4hX2hgLeR38Qn3oUZuk1SVs=",
+        "project_name": "endangeredtokens",
+        "project_namespace": "oso",
+        "project_source": "OSS_DIRECTORY",
+        "returning_address_count_90_days": 1,
+        "transaction_count": 22,
+        "transaction_count_6_months": 12
+      },
+      {
+        "active_contract_count_90_days": 2,
+        "address_count": 359,
+        "address_count_90_days": 2,
+        "days_since_first_transaction": 714,
+        "display_name": "Manifold",
+        "event_source": "OPTIMISM",
+        "gas_fees_sum": 0.10770561886011508,
+        "gas_fees_sum_6_months": 0.00043673927702894997,
+        "high_activity_address_count_90_days": 0,
+        "low_activity_address_count_90_days": 2,
+        "medium_activity_address_count_90_days": 0,
+        "multi_project_address_count_90_days": 0,
+        "new_address_count_90_days": 2,
+        "project_id": "a5hVSzyhsfqMcoWljR1DzjmwLFlDEhnU08YVni8xrPw=",
+        "project_name": "manifold",
+        "project_namespace": "oso",
+        "project_source": "OSS_DIRECTORY",
+        "returning_address_count_90_days": 0,
+        "transaction_count": 1012,
+        "transaction_count_6_months": 6
+      }
+    ]
   }
 }

packages/nextjs/.env.example

@@ -13,6 +13,7 @@ NEXT_PUBLIC_ALCHEMY_API_KEY=
 NEXT_PUBLIC_WALLET_CONNECT_PROJECT_ID=
 
 OSO_GRAPHQL_ENDPOINT=
+OSO_API_KEY=
 AGORA_API_ENDPOINT=https://vote.optimism.io/api_v1
 AGORA_API_KEY=
 

packages/nextjs/app/types/OSO.ts

@@ -1,47 +1,46 @@
 export interface OSOResponse {
-  onchain_metrics_by_project: OnchainMetricsByProject[];
-  code_metrics_by_project: CodeMetricsByProject[];
+  onchain_metrics_by_project_v1: OnchainMetricsByProject[];
 }
 
+// This data is easy to get right now so we can use it for testing if necessary - We will remove later when the actual data is available
 export interface OnchainMetricsByProject {
-  active_users: number;
-  first_txn_date: string;
-  high_frequency_users: number;
-  l2_gas_6_months: number;
-  less_active_users: number;
-  more_active_users: number;
-  multi_project_users: number;
-  network: string;
-  new_user_count: number;
-  num_contracts: number;
+  active_contract_count_90_days: number;
+  address_count: number;
+  address_count_90_days: number;
+  days_since_first_transaction: number;
+  display_name: string;
+  event_source: string;
+  gas_fees_sum: number;
+  gas_fees_sum_6_months: number;
+  high_activity_address_count_90_days: number;
+  low_activity_address_count_90_days: number;
+  medium_activity_address_count_90_days: number;
+  multi_project_address_count_90_days: number;
+  new_address_count_90_days: number;
   project_id: string;
   project_name: string;
-  total_l2_gas: number;
-  total_txns: number;
-  total_users: number;
-  txns_6_months: number;
-  users_6_months: number;
+  project_namespace: string;
+  project_source: string;
+  returning_address_count_90_days: number;
+  transaction_count: number;
+  transaction_count_6_months: number;
 }
 
-export interface CodeMetricsByProject {
-  avg_active_devs_6_months: number;
-  avg_fulltime_devs_6_months: number;
-  commits_6_months: number;
-  contributors: number;
-  contributors_6_months: number;
-  first_commit_date: string;
-  forks: number;
-  issues_closed_6_months: number;
-  issues_opened_6_months: any;
-  last_commit_date: string;
-  new_contributors_6_months: number;
+// These are the actual metrics we will be using when the data is provided by OSO
+export interface RF4ImpactMetricsByProject {
   project_id: string;
   project_name: string;
-  pull_requests_merged_6_months: number;
-  pull_requests_opened_6_months: number;
-  repositories: number;
-  source: string;
-  stars: number;
+  check_oss_requirements: boolean;
+  gas_fees: number; //**Gas Fees**: Sum of a project's total contribution to gas fees across the Superchain. Gas fees are the primary recurring revenue source for the Superchain and a key indicator of aggregate blockspace demand. A project’s gas fee contribution is influenced by its total volume of contract interactions, the computational complexity of those interactions, and the state of the underlying gas market at the time of those transactions. In the long run, gas fees are what will power Retro Funding and enable it to continue in perpetuity. All members of the Superchain have committed at least 15% of their gross profit from gas fees to Retro Funding. Supporting projects that generate revenue in the form of gas fees helps power the economic engine of the Superchain.
+  transaction_count: number; //**Total Transactions**: Count of a project’s transactions over the RF4 scope period October 2023 - June 2024. Optimism is a Layer 2 roll-up designed to improve the transaction throughput and reduce the fees on Ethereum. Layer 2s are crucial for scaling Ethereum because they help address the network's congestion issues without compromising its security or decentralization. Transaction counts are an important indicator for assessing the adoption and usage of all the new blockspace made available by the Superchain. Projects that have a sustained, high transaction count provide a clear signal of network growth and blockspace demand.
+  trusted_transaction_count: number; //**Interactions from Trusted Optimism Users**: Count of a project’s transactions performed by trusted users, over the RF4 scope period October 2023 - June 2024. Bots, airdrop farming, and sybil attacks are longstanding problems in crypto. There are several teams in the Optimism ecosystem building reputation models for labeling “trusted users” in a privacy-preserving way. This metric aggregates reputation data from multiple platforms ([Farcaster](https://docs.farcaster.xyz/learn/architecture/hubs), [Passport](https://www.passport.xyz/), [EigenTrust by Karma3Labs](https://docs.karma3labs.com/eigentrust)) and only considers transactions that come from trusted users, a small subset of all active addresses on the Superchain. By tracking interactions specifically from trusted users, we gain a picture of blockspace demand that is less influenced by the effects of bots / farmers / sybils.
+  trusted_transaction_share: number; //**Trusted Optimism Users' Share of Total Interactions**: Percentage of a project's total transactions that were made by trusted users over the RF4 scope period (October 2023 - June 2024). This metric expresses Interactions from Trusted Optimism Users and Total Transactions as a simple ratio. Using a ratio makes it easier to compare trusted user levels across big projects and small projects side-by-side. For example, a project with 10K trusted transactions out of 20K total transactions would score better than a project with 10K trusted transactions out of 50K total transactions. This indicator is nuanced because it recognizes that minimizing bot / farming / sybil activity might go against economic incentives in the short term but is important for network quality in the long term. Given that this indicator is calculated on a percentage basis, projects with fewer than 100 users are not evaluated.
+  trusted_users_onboarded: number; //**Users Onboarded**: Count of trusted users to the Superchain who were onboarded over the RF4 scope period (October 2023 - June 2024) and who interacted with a project within their first 30 days on the Superchain. Getting 1 billion users onchain won’t be easy. It will require better onramps and onchain UX than crypto natives are accustomed to. This metric identifies projects that helped onboard new users to the Superchain since October 2023. In order to qualify, a new user has to also be in the set of trusted users. Then, any project on any chain that a user interacted with in their first month on the Superchain is counted. This is often multiple projects per new user. Supporting projects that are the first port of call for new users is essential for expanding the size and reach of the Superchain user’s base.
+  daily_active_addresses: number; //**Average Daily Active Addresses (DAAs)**: Average of a project’s daily active addresses over the RF4 scope period (October 2023 - June 2024). Daily Active Addresses (DAAs) is a more granular view of a project's daily user activity and engagement levels than MAAs (Monthly Active Addresses). A high number of DAAs is a sign that Layer 2s have widespread adoption. While there is minimal cost to creating new addresses to farm a protocol, such farming or Sybil activity is usually short-lived. By averaging the number of active addresses on a daily basis over the RF4 period, this metric smooths out some of the blips and spikes in the data. New projects receive 0s for the days before they launched. Steady or rising DAAs over an extended period is a good signal of widespread adoption.
+  trusted_daily_active_users: number; //**Average Trusted Daily Active Users (DAUs)**: Average of a project’s daily active users (trusted users only) over the RF4 scope period (October 2023 - June 2024). Daily Active Users (DAUs) is a more granular view of a project's daily user activity and engagement levels than MAUs (Monthly Active Users). A high number of trusted DAUs would be a sign that Layer 2s have widespread adoption. The reality today is that there are very few apps that generate high levels of daily, revenue-generating activity from users. By averaging the number of active users on a daily basis over the RF4 period, this metric smooths out some of the blips and spikes in the data. New projects receive 0s for the days before they launched. Indeed, trusted DAUs is a hard metric to crack, but it truly hones in on projects that give their users a reason to come back frequently.
+  monthly_active_addresses: number; //**Average Monthly Active Addresses (MAAs)**: Average of a project’s monthly active addresses over the RF4 scope period (October 2023 - June 2024). Not all projects have lots of daily users. Some projects are more like utilities that are used once a month or at less regular intervals. Monthly Active Addresses (MAAs) is a key metric for understanding the size and engagement of a project’s user base over a more extended period. One word of cauation: as there is minimal cost to creating new addresses, MAAs may be more susceptible to farming than other metrics like Daily Active Addresses or ones that look only at "trusted users". However, by averaging the number of active addresses on a monthly basis over the RF4 period, this metric does smooth out some of the blips and spikes in the data. New projects receive 0s for the months before they launched.
+  trusted_monthly_active_users: number; //**Average Trusted Monthly Active Users (MAUs)**: Average of a project’s monthly active users (trusted users only) over the RF4 scope period (October 2023 - June 2024). We all know that attention is fleeting, especially in crypto. MAUs is one of the most important metrics for any project looking to grow a large user base. A project’s average MAUs also provides insights into its ongoing popularity and relevance within the Optimism ecosystem. The metric is calculated by counting the number of distinct trusted users for each month included in the RF4 scope period and then averaging the monthly totals. Newer projects receive 0s for the months before they launched. A consistent or growing base of trusted MAUs is a sign that there is a healthy, thriving community around a project.
+  recurring_addresses: number; //**Recurring Addresses**: Count of addresses that have interacted with the project in at least 3 separate months over the RF4 scope period (October 2023 - June 2024). Recurring addresses are a proxy for recurring users. It is especially relevant to projects where users may explicitly choose to interact with the project from a distinct addresses in order to preserve their privacy. By counting the number of distinct addresses that have interacted with a project over the course of at least three distinct calendar months during the RF4 scope period, this metric provides a view of user quality that complements metrics derived from the "trusted user" model. A high count of recurring addresses signals strong project loyalty and a good user experience.
+  trusted_recurring_users: number; //**Trusted Recurring Users**: Count of trusted users who have interacted with the project in at least 3 separate months over the RF4 scope period (October 2023 - June 2024). Many crypto natives are curious to try out new protocols. But churn and user retention are major issues. Recurring users represent the most loyal and committed segment of a project's user base. This metric considers users who have interacted with a project over the course of at least three distinct calendar months during the RF4 scope period. Thus, it is intended to reflect sustained interest and ongoing engagement over time. A high count of recurring users signals strong project loyalty and a good user experience, and helps separate the fads from the future.
+  power_user_addresses: number; //**Power User Addresses**: Count of 'power user' addresses that have interacted with the project over the RF4 scope period (October 2023 - June 2024). This metric reflects the degree which a project has attracted attention from the most active and engaged users on the Superchain. A `power user` is defined as an address that has made at least 100 transactions, across at least 10 different projects, on at least 30 days, over the RF4 scope period. A project is counted by this metric if has at least one interaction from a power user. Power users are critical early adopters for the ecosystem.
 }
-
-export type OSOResponseProps = OnchainMetricsByProject | CodeMetricsByProject;

packages/nextjs/pages/api/etl/index.ts

@@ -0,0 +1,31 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  if (req.method !== "GET") {
+    return res.status(405).json({ message: "Method not allowed." });
+  }
+  // Accept Date params for ease of backfilling
+
+  // Check if process is running/already run for today
+  // If not, run process - set process as 'running' in DB
+
+  // Get data from OSO
+  // Save raw response to DB (so that we can recreate data if needed)
+
+  // Get Agora Data
+  // Check for new projects inside the returned data
+
+  // Run ETL process without awaiting
+
+  // Go ahead and return success message or already running/already run message
+  try {
+    if (true) {
+      res.status(200).json({ message: "ETL process started for dd/mm/yyyy" });
+    } else {
+      res.status(404).json({ message: "Vectors not found" });
+    }
+  } catch (error) {
+    console.error(error);
+    res.status(500).json({ message: "Internal Server Error" });
+  }
+}

packages/nextjs/services/openSourceObserver/OSOFetch.ts

@@ -2,57 +2,46 @@ import { OSOResponse } from "~~/app/types/OSO";
 
 export const fetchOSOProjectData = async () => {
   const query = `{
-      onchain_metrics_by_project(distinct_on: project_id, where: {network: {_eq: "OPTIMISM"}}) {
-        active_users
-        first_txn_date
-        high_frequency_users
-        l2_gas_6_months
-        less_active_users
-        more_active_users
-        multi_project_users
-        network
-        new_user_count
-        num_contracts
+      onchain_metrics_by_project_v1(where: {event_source: {_eq: "OPTIMISM"}}) {
+        active_contract_count_90_days
+        address_count
+        address_count_90_days
+        days_since_first_transaction
+        display_name
+        event_source
+        gas_fees_sum
+        gas_fees_sum_6_months
+        high_activity_address_count_90_days
+        low_activity_address_count_90_days
+        medium_activity_address_count_90_days
+        multi_project_address_count_90_days
+        new_address_count_90_days
         project_id
         project_name
-        total_l2_gas
-        total_txns
-        total_users
-        txns_6_months
-        users_6_months
-      }
-      code_metrics_by_project(distinct_on: project_id) {
-        avg_active_devs_6_months
-        avg_fulltime_devs_6_months
-        commits_6_months
-        contributors
-        contributors_6_months
-        first_commit_date
-        forks
-        issues_closed_6_months
-        issues_opened_6_months
-        last_commit_date
-        new_contributors_6_months
-        project_id
-        project_name
-        pull_requests_merged_6_months
-        pull_requests_opened_6_months
-        repositories
-        source
-        stars
+        project_namespace
+        project_source
+        returning_address_count_90_days
+        transaction_count
+        transaction_count_6_months
       }
     }`;
 
   const OSOGraphQLEndpoint = process.env.OSO_GRAPHQL_ENDPOINT as string;
+  const OSOApiKey = process.env.OSO_API_KEY as string;
 
   if (!OSOGraphQLEndpoint) {
     throw new Error("OSO_GRAPHQL_ENDPOINT env var is not defined");
   }
 
+  if (!OSOApiKey) {
+    throw new Error("OSO_API_KEY env var is not defined");
+  }
+
   const response = await fetch(OSOGraphQLEndpoint, {
     method: "POST",
     headers: {
       "Content-Type": "application/json",
+      Authorization: `Bearer ${OSOApiKey}`,
     },
     body: JSON.stringify({ query }),
   });