add uploads

.github/workflows/upload_to_dune.yml

@@ -0,0 +1,33 @@
+name: Upload CSVs to Dune on Commit
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'queries/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Log directory structure
+        run: |
+          pwd
+          ls -R
+    
+      - name: pip requirements
+        run: pip install -r requirements.txt
+
+      - name: Update all queries from Dune, by overwriting queries with repo query text
+        env:
+          DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
+          DUNE_API_BASE_URL: ${{ secrets.DUNE_API_BASE_URL }}
+        run: python -u scripts/upload_to_dune.py

queries/queries.yml

@@ -1,3 +1,2 @@
 query_ids:
-  - 2615782
-  - 2615781
+  - 123456

requirements.txt

@@ -1,4 +1,4 @@
-dune-client==1.3.0
+dune-client
 pyyaml
 python-dotenv
 pandas

scripts/push_to_dune.py

@@ -36,13 +36,14 @@
         with open(file_path, 'r', encoding='utf-8') as file:
             text = file.read()
 
-        # Update existing file
-        dune.update_query(
-            query.base.query_id, 
-            # All parameters below are optional
-            query_sql=text,
-        )
-        print('SUCCESS: updated query {} to dune'.format(query.base.query_id))
-
+        try:
+            # Update existing file
+            dune.update_query(
+                query.base.query_id, 
+                query_sql=text,
+            )
+            print('SUCCESS: updated query {} to dune'.format(query.base.query_id))
+        except Exception as e:
+            print('ERROR: {}'.format(str(e))) #likely API permission errors
     else:
         print('ERROR: file not found, query id {}'.format(query.base.query_id))

scripts/upload_to_dune.py

@@ -0,0 +1,33 @@
+import os
+from dune_client.client import DuneClient
+from dotenv import load_dotenv
+import sys
+import codecs
+import os
+
+# Set the default encoding to UTF-8
+sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env')
+load_dotenv(dotenv_path)
+
+dune = DuneClient.from_env()
+
+uploads_path = os.path.join(os.path.dirname(__file__), '..', 'uploads')
+files = os.listdir(uploads_path)
+
+for file in files:
+    if not file.endswith(".csv"):
+        continue
+    file_name = file.split(".")[0].lower().replace(' ', '_')
+    with open(os.path.join(uploads_path, file), 'r') as file:
+        try:
+            table = dune.upload_csv(
+                data=str(file.read()),
+                table_name=file_name,
+                is_private=False
+            )
+            print(f'uploaded table "{file_name}"')
+        except Exception as e:
+            print(f"An error occurred while uploading the CSV: {str(e)}")
+