move around

.env.test

@@ -1,4 +1,5 @@
-#copy and paste this file into a .env file to run scripts locally. You will also need to add the DUNE_API_KEY into the repo settings under "Secrets and Variables"
+#copy and paste this file into a .env file to run scripts locally. You will also need to add the DUNE_API_KEY and DUNE_API_BASE_URL into the repo settings under "Secrets and Variables"
 
 #add a dune API key - you can create one under team settings (https://dune.com/settings/teams/manage/{team_name}/api). You must be on the premium plan.
-DUNE_API_KEY=
+DUNE_API_KEY=
+DUNE_API_BASE_URL=https://api.dune.com

.github/workflows/upload_to_dune.yml

@@ -0,0 +1,33 @@
+name: Upload CSVs to Dune on Commit
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'queries/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Log directory structure
+        run: |
+          pwd
+          ls -R
+    
+      - name: pip requirements
+        run: pip install -r requirements.txt
+
+      - name: Update all queries from Dune, by overwriting queries with repo query text
+        env:
+          DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }}
+          DUNE_API_BASE_URL: ${{ secrets.DUNE_API_BASE_URL }}
+        run: python -u scripts/upload_to_dune.py

README.md

@@ -1,18 +1,22 @@
 # Dune Query Repo
 
-A template for creating repos to manage your Dune queries (using the [Dune CRUD API](https://dune.mintlify.app/api-reference/crud/endpoint/create)). The main flow I've created this template for is to turn any dashboard you own into a repository of queries. But you can extend it however you want.
+A template for creating repos to [manage your Dune queries](https://dune.mintlify.app/api-reference/crud/endpoint/create) and any [CSVs as Dune tables](https://dune.mintlify.app/api-reference/upload/endpoint/upload). The main flow I've created this template for is to turn any dashboard you own into a repository of queries. But you can extend it however you want.
 
 ### Setup Your Repo
 
-1. Generate an API key from your Dune account and put that in both your `.env` file and [github action secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository) (name it `DUNE_API_KEY`). You can create a key under your Dune team settings. *The key must be from a premium plan for this repo to work.*
+1. Generate an API key from your Dune account and put that in both your `.env` file and [github action secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository) (name it `DUNE_API_KEY`). You can create a key under your Dune team settings. Add `https://api.dune.com/api` into the action secrets under `DUNE_API_BASE_URL` too. *The api key must be from a plus plan for this repo to work.*
 
 2. Then, go to the dashboard you want to create a repo for (must be owned by you/your team). Click on the "github" button in the top right of your dashboard to see the query ids.
 
-3. Copy and paste that list into the `queries.yml` file inside the `/queries` folder. You can paste any list of query ids, doesn't have to be linked to a dashboard.
+3. Copy and paste that list into the `queries.yml` file. You can paste any list of query ids, it doesn't have to be linked to a dashboard.
 
 4. Then, run `pull_from_dune.py` to bring in all queries into `/query_{id}.sql` files within the `/queries` folder. Directions to setup and run this script are below.
 
-5. Make any changes you need to directly in the repo. Any time you push a commit to MAIN branch, `push_to_dune.py` will save your changes into Dune directly.
+### Updating Queries or CSV Tables
+
+1. Make any changes you need to directly in the repo. Any time you push a commit to MAIN branch, `push_to_dune.py` will save your changes into Dune directly. You can run this manually too if you want.
+
+2. For CSVs, update the files in the `/uploads` folder. `upload_to_dune.py` will run on commit, or can be run manually. The table name in Dune will be `dune.team_name.dataset_<filename>`.
 
 ---
 
@@ -24,11 +28,12 @@ You'll need python and pip installed to run the script commands. If you don't ha
 pip install -r requirements.txt
 ```
 
-| Script | Action | Command |
-|---|---|---|
-| `pull_from_dune.py` | updates/adds queries to your repo based on ids in `queries.yml` | `python scripts/pull_from_dune.py` |
-| `push_to_dune.py` | updates queries to Dune based on files in your `/queries` folder | `python scripts/push_to_dune.py` |
-| `preview_query.py` | gives you the first 20 rows of results by running a query from your `/queries` folder. Specify the id. | `python scripts/preview_query.py 2615782` |
+| Script | Action                                                                                                                                                    | Command |
+|---|-----------------------------------------------------------------------------------------------------------------------------------------------------------|---|
+| `pull_from_dune.py` | updates/adds queries to your repo based on ids in `queries.yml`                                                                                           | `python scripts/pull_from_dune.py` |
+| `push_to_dune.py` | updates queries to Dune based on files in your `/queries` folder                                                                                          | `python scripts/push_to_dune.py` |
+| `preview_query.py` | gives you the first 20 rows of results by running a query from your `/queries` folder. Specify the id. This uses Dune API credits | `python scripts/preview_query.py 2615782` |
+| `upload_to_dune.py` | uploads/updates any tables from your `/uploads` folder. Must be in CSV format, and under 200MB. | `python scripts/upload_to_dune.py` |
 
 ---
 

queries.yml

@@ -0,0 +1,14 @@
+query_ids:
+  - 4235
+  - 8243
+  - 14138
+  - 7486
+  - 4319
+  - 21693
+  - 4323
+  - 1847
+  - 4388
+  - 2180075
+  - 4424
+  - 21689
+  - 4234

requirements.txt

@@ -1,4 +1,4 @@
-dune-client==1.3.0
+dune-client
 pyyaml
 python-dotenv
 pandas

scripts/preview_query.py

@@ -24,9 +24,9 @@
     with open(query_file, 'r', encoding='utf-8') as file:
         query_text = file.read()
 
-    print(query_text + '\nlimit 20')
+    print('select * from (\n' + query_text + '\n) limit 20')
 
-    results = dune.run_sql(query_text + '\nlimit 20')
+    results = dune.run_sql('select * from (\n' + query_text + '\n) limit 20')
     # print(results.result.rows)
     results = pd.DataFrame(data=results.result.rows)
     print('\n')
@@ -36,4 +36,4 @@
     print('\n')
     print(results.info())
 else:
-    print('query id file not found, try again')
+    print('query id file not found, try again')

scripts/pull_from_dune.py

@@ -14,7 +14,7 @@
 dune = DuneClient.from_env()
 
 # Read the queries.yml file
-queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries', 'queries.yml')
+queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries.yml')
 with open(queries_yml, 'r', encoding='utf-8') as file:
     data = yaml.safe_load(file)
 

scripts/push_to_dune.py

@@ -14,35 +14,40 @@
 dune = DuneClient.from_env()
 
 # Read the queries.yml file
-queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries', 'queries.yml')
+queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries.yml')
 with open(queries_yml, 'r', encoding='utf-8') as file:
     data = yaml.safe_load(file)
 
 # Extract the query_ids from the data
 query_ids = [id for id in data['query_ids']]
+print(query_ids)
 
 for id in query_ids:
-    query = dune.get_query(id)
-    print('PROCESSING: query {}, {}'.format(query.base.query_id, query.base.name))
-
-    # Check if query file exists in /queries folder
-    queries_path = os.path.join(os.path.dirname(__file__), '..', 'queries')
-    files = os.listdir(queries_path)
-    found_files = [file for file in files if str(id) == file.split('___')[-1].split('.')[0]]
-
-    if len(found_files) != 0:
-        file_path = os.path.join(os.path.dirname(__file__), '..', 'queries', found_files[0])
-        # Read the content of the file
-        with open(file_path, 'r', encoding='utf-8') as file:
-            text = file.read()
-
-        # Update existing file
-        dune.update_query(
-            query.base.query_id, 
-            # All parameters below are optional
-            query_sql=text,
-        )
-        print('SUCCESS: updated query {} to dune'.format(query.base.query_id))
+    try:
+        query = dune.get_query(id)
+        print('PROCESSING: query {}, {}'.format(query.base.query_id, query.base.name))
+
+        # Check if query file exists in /queries folder
+        queries_path = os.path.join(os.path.dirname(__file__), '..', 'queries')
+        files = os.listdir(queries_path)
+        found_files = [file for file in files if str(id) == file.split('___')[-1].split('.')[0]]
 
-    else:
-        print('ERROR: file not found, query id {}'.format(query.base.query_id))
+        if len(found_files) != 0:
+            file_path = os.path.join(os.path.dirname(__file__), '..', 'queries', found_files[0])
+            # Read the content of the file
+            with open(file_path, 'r', encoding='utf-8') as file:
+                text = file.read()
+
+            # Update existing file
+            dune.update_query(
+                query.base.query_id, 
+                # All parameters below are optional
+                query_sql=text,
+            )
+            print('SUCCESS: updated query {} to dune'.format(query.base.query_id))
+
+        else:
+            print('ERROR: file not found, query id {}'.format(query.base.query_id))
+    except Exception as e:
+        print('ERROR: {}'.format(str(e))) #likely API permission errors
+        print('Most errors are because your API key was not generated under the context of the query owner.')

scripts/upload_to_dune.py

@@ -0,0 +1,30 @@
+import os
+from dune_client.client import DuneClient
+from dotenv import load_dotenv
+import sys
+import codecs
+import os
+
+# Set the default encoding to UTF-8
+sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+
+dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env')
+load_dotenv(dotenv_path)
+
+dune = DuneClient.from_env()
+
+uploads_path = os.path.join(os.path.dirname(__file__), '..', 'uploads')
+files = os.listdir(uploads_path)
+
+for file in files:
+    if not file.endswith(".csv"):
+        continue
+    file_name = file.split(".")[0].lower().replace(' ', '_')
+    with open(os.path.join(uploads_path, file), 'r') as file:
+        table = dune.upload_csv(
+            data=str(file.read()),
+            table_name=file_name,
+            is_private=False
+        )
+        print(f'uploaded table "{file_name}"')
+