diff --git a/.env.test b/.env.test index 0145f72..ec22d0e 100644 --- a/.env.test +++ b/.env.test @@ -1,4 +1,5 @@ -#copy and paste this file into a .env file to run scripts locally. You will also need to add the DUNE_API_KEY into the repo settings under "Secrets and Variables" +#copy and paste this file into a .env file to run scripts locally. You will also need to add the DUNE_API_KEY and DUNE_API_BASE_URL into the repo settings under "Secrets and Variables" #add a dune API key - you can create one under team settings (https://dune.com/settings/teams/manage/{team_name}/api). You must be on the premium plan. -DUNE_API_KEY= \ No newline at end of file +DUNE_API_KEY= +DUNE_API_BASE_URL=https://api.dune.com \ No newline at end of file diff --git a/.github/workflows/upload_to_dune.yml b/.github/workflows/upload_to_dune.yml new file mode 100644 index 0000000..4f079de --- /dev/null +++ b/.github/workflows/upload_to_dune.yml @@ -0,0 +1,33 @@ +name: Upload CSVs to Dune on Commit + +on: + push: + branches: + - main + paths: + - 'queries/**' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Log directory structure + run: | + pwd + ls -R + + - name: pip requirements + run: pip install -r requirements.txt + + - name: Update all queries from Dune, by overwriting queries with repo query text + env: + DUNE_API_KEY: ${{ secrets.DUNE_API_KEY }} + DUNE_API_BASE_URL: ${{ secrets.DUNE_API_BASE_URL }} + run: python -u scripts/upload_to_dune.py diff --git a/README.md b/README.md index a781946..f7edeea 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,22 @@ # Dune Query Repo -A template for creating repos to manage your Dune queries (using the [Dune CRUD API](https://dune.mintlify.app/api-reference/crud/endpoint/create)). The main flow I've created this template for is to turn any dashboard you own into a repository of queries. But you can extend it however you want. +A template for creating repos to [manage your Dune queries](https://dune.mintlify.app/api-reference/crud/endpoint/create) and any [CSVs as Dune tables](https://dune.mintlify.app/api-reference/upload/endpoint/upload). The main flow I've created this template for is to turn any dashboard you own into a repository of queries. But you can extend it however you want. ### Setup Your Repo -1. Generate an API key from your Dune account and put that in both your `.env` file and [github action secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository) (name it `DUNE_API_KEY`). You can create a key under your Dune team settings. *The key must be from a premium plan for this repo to work.* +1. Generate an API key from your Dune account and put that in both your `.env` file and [github action secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions#creating-secrets-for-a-repository) (name it `DUNE_API_KEY`). You can create a key under your Dune team settings. Add `https://api.dune.com/api` into the action secrets under `DUNE_API_BASE_URL` too. *The api key must be from a plus plan for this repo to work.* 2. Then, go to the dashboard you want to create a repo for (must be owned by you/your team). Click on the "github" button in the top right of your dashboard to see the query ids. -3. Copy and paste that list into the `queries.yml` file inside the `/queries` folder. You can paste any list of query ids, doesn't have to be linked to a dashboard. +3. Copy and paste that list into the `queries.yml` file. You can paste any list of query ids, it doesn't have to be linked to a dashboard. 4. Then, run `pull_from_dune.py` to bring in all queries into `/query_{id}.sql` files within the `/queries` folder. Directions to setup and run this script are below. -5. Make any changes you need to directly in the repo. Any time you push a commit to MAIN branch, `push_to_dune.py` will save your changes into Dune directly. +### Updating Queries or CSV Tables + +1. Make any changes you need to directly in the repo. Any time you push a commit to MAIN branch, `push_to_dune.py` will save your changes into Dune directly. You can run this manually too if you want. + +2. For CSVs, update the files in the `/uploads` folder. `upload_to_dune.py` will run on commit, or can be run manually. The table name in Dune will be `dune.team_name.dataset_`. --- @@ -24,11 +28,12 @@ You'll need python and pip installed to run the script commands. If you don't ha pip install -r requirements.txt ``` -| Script | Action | Command | -|---|---|---| -| `pull_from_dune.py` | updates/adds queries to your repo based on ids in `queries.yml` | `python scripts/pull_from_dune.py` | -| `push_to_dune.py` | updates queries to Dune based on files in your `/queries` folder | `python scripts/push_to_dune.py` | -| `preview_query.py` | gives you the first 20 rows of results by running a query from your `/queries` folder. Specify the id. | `python scripts/preview_query.py 2615782` | +| Script | Action | Command | +|---|-----------------------------------------------------------------------------------------------------------------------------------------------------------|---| +| `pull_from_dune.py` | updates/adds queries to your repo based on ids in `queries.yml` | `python scripts/pull_from_dune.py` | +| `push_to_dune.py` | updates queries to Dune based on files in your `/queries` folder | `python scripts/push_to_dune.py` | +| `preview_query.py` | gives you the first 20 rows of results by running a query from your `/queries` folder. Specify the id. This uses Dune API credits | `python scripts/preview_query.py 2615782` | +| `upload_to_dune.py` | uploads/updates any tables from your `/uploads` folder. Must be in CSV format, and under 200MB. | `python scripts/upload_to_dune.py` | --- diff --git a/queries.yml b/queries.yml new file mode 100644 index 0000000..4e37447 --- /dev/null +++ b/queries.yml @@ -0,0 +1,14 @@ +query_ids: + - 4235 + - 8243 + - 14138 + - 7486 + - 4319 + - 21693 + - 4323 + - 1847 + - 4388 + - 2180075 + - 4424 + - 21689 + - 4234 diff --git a/queries/queries.yml b/queries/queries.yml deleted file mode 100644 index 4991afd..0000000 --- a/queries/queries.yml +++ /dev/null @@ -1,14 +0,0 @@ -query_ids: -- 4235 -- 8243 -- 14138 -- 7486 -- 4319 -- 21693 -- 4323 -- 1847 -- 4388 -- 2180075 -- 4424 -- 21689 -- 4234 diff --git a/requirements.txt b/requirements.txt index ad76048..6888d99 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -dune-client==1.3.0 +dune-client pyyaml python-dotenv pandas \ No newline at end of file diff --git a/scripts/preview_query.py b/scripts/preview_query.py index 0ba50f2..0268db6 100644 --- a/scripts/preview_query.py +++ b/scripts/preview_query.py @@ -24,9 +24,9 @@ with open(query_file, 'r', encoding='utf-8') as file: query_text = file.read() - print(query_text + '\nlimit 20') + print('select * from (\n' + query_text + '\n) limit 20') - results = dune.run_sql(query_text + '\nlimit 20') + results = dune.run_sql('select * from (\n' + query_text + '\n) limit 20') # print(results.result.rows) results = pd.DataFrame(data=results.result.rows) print('\n') @@ -36,4 +36,4 @@ print('\n') print(results.info()) else: - print('query id file not found, try again') \ No newline at end of file + print('query id file not found, try again') diff --git a/scripts/pull_from_dune.py b/scripts/pull_from_dune.py index 39fa8e5..9b60313 100644 --- a/scripts/pull_from_dune.py +++ b/scripts/pull_from_dune.py @@ -14,7 +14,7 @@ dune = DuneClient.from_env() # Read the queries.yml file -queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries', 'queries.yml') +queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries.yml') with open(queries_yml, 'r', encoding='utf-8') as file: data = yaml.safe_load(file) diff --git a/scripts/push_to_dune.py b/scripts/push_to_dune.py index 71b1932..a81fe7b 100644 --- a/scripts/push_to_dune.py +++ b/scripts/push_to_dune.py @@ -14,35 +14,40 @@ dune = DuneClient.from_env() # Read the queries.yml file -queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries', 'queries.yml') +queries_yml = os.path.join(os.path.dirname(__file__), '..', 'queries.yml') with open(queries_yml, 'r', encoding='utf-8') as file: data = yaml.safe_load(file) # Extract the query_ids from the data query_ids = [id for id in data['query_ids']] +print(query_ids) for id in query_ids: - query = dune.get_query(id) - print('PROCESSING: query {}, {}'.format(query.base.query_id, query.base.name)) - - # Check if query file exists in /queries folder - queries_path = os.path.join(os.path.dirname(__file__), '..', 'queries') - files = os.listdir(queries_path) - found_files = [file for file in files if str(id) == file.split('___')[-1].split('.')[0]] - - if len(found_files) != 0: - file_path = os.path.join(os.path.dirname(__file__), '..', 'queries', found_files[0]) - # Read the content of the file - with open(file_path, 'r', encoding='utf-8') as file: - text = file.read() - - # Update existing file - dune.update_query( - query.base.query_id, - # All parameters below are optional - query_sql=text, - ) - print('SUCCESS: updated query {} to dune'.format(query.base.query_id)) + try: + query = dune.get_query(id) + print('PROCESSING: query {}, {}'.format(query.base.query_id, query.base.name)) + + # Check if query file exists in /queries folder + queries_path = os.path.join(os.path.dirname(__file__), '..', 'queries') + files = os.listdir(queries_path) + found_files = [file for file in files if str(id) == file.split('___')[-1].split('.')[0]] - else: - print('ERROR: file not found, query id {}'.format(query.base.query_id)) + if len(found_files) != 0: + file_path = os.path.join(os.path.dirname(__file__), '..', 'queries', found_files[0]) + # Read the content of the file + with open(file_path, 'r', encoding='utf-8') as file: + text = file.read() + + # Update existing file + dune.update_query( + query.base.query_id, + # All parameters below are optional + query_sql=text, + ) + print('SUCCESS: updated query {} to dune'.format(query.base.query_id)) + + else: + print('ERROR: file not found, query id {}'.format(query.base.query_id)) + except Exception as e: + print('ERROR: {}'.format(str(e))) #likely API permission errors + print('Most errors are because your API key was not generated under the context of the query owner.') diff --git a/scripts/upload_to_dune.py b/scripts/upload_to_dune.py new file mode 100644 index 0000000..085be44 --- /dev/null +++ b/scripts/upload_to_dune.py @@ -0,0 +1,30 @@ +import os +from dune_client.client import DuneClient +from dotenv import load_dotenv +import sys +import codecs +import os + +# Set the default encoding to UTF-8 +sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) + +dotenv_path = os.path.join(os.path.dirname(__file__), '..', '.env') +load_dotenv(dotenv_path) + +dune = DuneClient.from_env() + +uploads_path = os.path.join(os.path.dirname(__file__), '..', 'uploads') +files = os.listdir(uploads_path) + +for file in files: + if not file.endswith(".csv"): + continue + file_name = file.split(".")[0].lower().replace(' ', '_') + with open(os.path.join(uploads_path, file), 'r') as file: + table = dune.upload_csv( + data=str(file.read()), + table_name=file_name, + is_private=False + ) + print(f'uploaded table "{file_name}"') +