makes explorer resilient against datasets downtime

flashbots · Jul 9, 2024 · 64dec57 · 64dec57
1 parent 28b1922
commit 64dec57
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 40 deletions.
diff --git a/app/components/Metrics/Metrics.tsx b/app/components/Metrics/Metrics.tsx
@@ -89,10 +89,8 @@ const Metrics: React.FC = () => {
         const medianRefundValueUsd = medianRefundValueEth * ethUsdPriceToday;
 
         // use latest data (as oppose to 30 day median) for txes + users
-        const latestDate = new Date();
-        latestDate.setDate(latestDate.getDate() - 6); // data from datasets is often 2 days old
-        const latestDateString = latestDate.toISOString().split('T')[0];
-        const latestData = state.data.filter((d: any) => d.date === latestDateString);
+        const latestDate = state.latestDateFetched;
+        const latestData = state.data.filter((d: any) => d.date === latestDate);
 
         const intervalTiming = 12 // for 12s slots
         const numTxes = latestData.length;

diff --git a/app/components/Table/Table.tsx b/app/components/Table/Table.tsx
@@ -26,18 +26,9 @@ const Table: React.FC = () => {
     const fetchData = async () => {
       dispatch({ type: 'FETCH_DATA_START' });
       try {
-        // Fetch the latest data first
-        await fetchCSV(getDateRange('latest'));
-
-        // Fetch the last 30 days of data
-        await fetchCSV(getDateRange('last_30_days'));
-
-        // Fetch the Clickhouse data
+        await fetchCSV(90);
         const clickhouseData = await fetchClickhouseClient();
         dispatch({ type: 'FETCH_CLICKHOUSE_DATA_SUCCESS', payload: clickhouseData });
-
-        // Continue fetching up to 90 days of data for AddressChecker
-        fetchCSV(getDateRange('last_90_days'));
       } catch (error) {
         if (error instanceof Error) {
           dispatch({ type: 'FETCH_DATA_FAILURE', payload: error.message });
@@ -59,34 +50,17 @@ const Table: React.FC = () => {
     setCurrentPage(page);
   };
 
-  const getDateRange = (range: 'latest' | 'last_30_days' | 'last_90_days'): string[] => {
-    const dates: string[] = [];
-    const today = new Date();
-    const days = range === 'latest' ? 6 : range === 'last_30_days' ? 30 : 90; 
-
-    for (let i = 6; i < days; i++) {
-      const date = new Date(today);
-      date.setDate(today.getDate() - i); // latest data is often two days old
-      dates.push(formatDate(date));
-    }
-    return dates;
-  };
-
-  const formatDate = (date: Date): string => {
-    const year = date.getFullYear();
-    const month = String(date.getMonth() + 1).padStart(2, '0');
-    const day = String(date.getDate()).padStart(2, '0');
-    return `${year}-${month}-${day}`;
-  };
-
   const getRecentTransactions = () => {
+    if (!state.latestDateFetched) return [];
+
+    const latestDate = new Date(state.latestDateFetched);
+
+    // Get transactions from the last 7 days (including the latest date)
     return state.data
       .filter((transaction) => {
         const transactionDate = new Date(transaction.block_time);
-        const today = new Date();
-        const twoDaysAgo = new Date(today);
-        twoDaysAgo.setDate(today.getDate() - 6);
-        return transactionDate >= twoDaysAgo && transactionDate <= today;
+        const daysDifference = (latestDate.getTime() - transactionDate.getTime()) / (1000 * 3600 * 24);
+        return daysDifference >= 0 && daysDifference < 7;
       })
       .sort((a, b) => new Date(b.block_time).getTime() - new Date(a.block_time).getTime());
   };

diff --git a/app/context/DataContext.tsx b/app/context/DataContext.tsx
@@ -6,6 +6,7 @@ type State = {
   loading: boolean;
   error: string | null;
   clickhouseData: { columns: any[]; rows: any[] };
+  latestDateFetched: string | null;
 };
 
 type Action =
@@ -14,14 +15,16 @@ type Action =
   | { type: 'FETCH_DATA_FAILURE'; payload: string }
   | { type: 'SET_ETH_PRICE'; payload: { date: string; price: number } }
   | { type: 'FETCH_CLICKHOUSE_DATA_SUCCESS'; payload: { columns: any[]; rows: any[] } }
-  | { type: 'FETCH_CLICKHOUSE_DATA_FAILURE'; payload: string };
+  | { type: 'FETCH_CLICKHOUSE_DATA_FAILURE'; payload: string }
+  | { type: 'SET_LATEST_DATE_FETCHED'; payload: string };
 
 const initialState: State = {
   data: [],
   ethPrices: {},
   loading: false,
   error: null,
   clickhouseData: { columns: [], rows: [] },
+  latestDateFetched: null,
 };
 
 const dataReducer = (state: State, action: Action): State => {
@@ -38,6 +41,8 @@ const dataReducer = (state: State, action: Action): State => {
       return { ...state, loading: false, clickhouseData: action.payload };
     case 'FETCH_CLICKHOUSE_DATA_FAILURE':
       return { ...state, loading: false, error: action.payload };
+    case 'SET_LATEST_DATE_FETCHED':
+      return { ...state, latestDateFetched: action.payload };
     default:
       return state;
   }

diff --git a/app/lib/fetchCSV.ts b/app/lib/fetchCSV.ts
@@ -17,9 +17,33 @@ export const useFetchCSV = () => {
     return parsedData.data.map((entry: any) => ({ ...entry, date }));
   };
 
-  const fetchCSV = async (dates: string[]): Promise<void> => {
+  const findLatestAvailableDate = async (): Promise<string | null> => {
+    const today = new Date();
+    for (let i = 2; i <= 10; i++) {
+      const date = new Date(today);
+      date.setDate(today.getDate() - i);
+      const formattedDate = formatDate(date);
+      try {
+        await fetchCSVForDate(formattedDate);
+        return formattedDate;
+      } catch (error) {
+        console.log(`No data available for ${formattedDate}`);
+      }
+    }
+    return null;
+  };
+
+  const fetchCSV = async (daysToFetch: number = 90): Promise<void> => {
     dispatch({ type: 'FETCH_DATA_START' });
     try {
+      const latestDate = await findLatestAvailableDate();
+      if (!latestDate) {
+        throw new Error('No data available for the last 7 days');
+      }
+
+      dispatch({ type: 'SET_LATEST_DATE_FETCHED', payload: latestDate });
+
+      const dates = getDatesArray(latestDate, daysToFetch);
       const data = await Promise.all(dates.map(date => fetchCSVForDate(date)));
       dispatch({ type: 'FETCH_DATA_SUCCESS', payload: data.flat() });
     } catch (error) {
@@ -28,4 +52,19 @@ export const useFetchCSV = () => {
   };
 
   return fetchCSV;
+};
+
+const formatDate = (date: Date): string => {
+  return date.toISOString().split('T')[0];
+};
+
+const getDatesArray = (startDate: string, daysToFetch: number): string[] => {
+  const dates: string[] = [];
+  const start = new Date(startDate);
+  for (let i = 0; i < daysToFetch; i++) {
+    const date = new Date(start);
+    date.setDate(start.getDate() - i);
+    dates.push(formatDate(date));
+  }
+  return dates;
 };