From 96dad6ac2fe0f583a20c1175d396b9df955b9178 Mon Sep 17 00:00:00 2001 From: Michael Whittle Date: Wed, 16 Aug 2023 16:27:57 +0100 Subject: [PATCH 1/2] fixed coinbase advanced trade missing candles issue --- .github/workflows/unit-tests.yml | 4 ++-- examples/script-coinbase.py | 6 ++---- models/exchange/coinbase/api.py | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 56ad6716..0a72aadb 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -17,10 +17,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 3.9 + - name: Set up Python 3.11 uses: actions/setup-python@v2 with: - python-version: 3.9 + python-version: 3.11 - name: Find typos with codespell uses: codespell-project/actions-codespell@master with: diff --git a/examples/script-coinbase.py b/examples/script-coinbase.py index 9ea91232..a1acb7df 100644 --- a/examples/script-coinbase.py +++ b/examples/script-coinbase.py @@ -10,11 +10,9 @@ app1 = PyCryptoBot(exchange="coinbase") model1 = CBAuthAPI(app1.api_key, app1.api_secret, app1.api_url, app=app1) -""" app2 = PyCryptoBot(exchange="coinbasepro") model2 = CAuthAPI(app2.api_key, app2.api_secret, app2.api_passphrase, app2.api_url, app=app2) model3 = CPublicAPI(app=app2) -""" """ COINBASE""" # df = model1.get_accounts() @@ -74,8 +72,8 @@ print(df) """ COINBASE PRO""" -# df = model3.get_historical_data("ADA-GBP", Granularity.ONE_HOUR) # Coinbase Pro has this in the public API, Advanced Trade has this in the auth API -# print(df) +df = model3.get_historical_data("ADA-GBP", Granularity.ONE_HOUR) # Coinbase Pro has this in the public API, Advanced Trade has this in the auth API +print(df) """ COINBASE""" diff --git a/models/exchange/coinbase/api.py b/models/exchange/coinbase/api.py index 0d12a208..b7d72812 100644 --- a/models/exchange/coinbase/api.py +++ b/models/exchange/coinbase/api.py @@ -740,6 +740,22 @@ def get_historical_data( df["close"] = df["close"].astype(float) df["volume"] = df["volume"].astype(float) + # create a full range of requested interval + full_range = pd.date_range(start=df.index[0], end=df.index[-1], freq=freq) + + # re-index the dataframe using the full range + df = df.reindex(full_range) + + # fill missing values and forward-fill the price columns and set volume to 0 for missing rows. + df["open"].fillna(method="ffill", inplace=True) + df["high"].fillna(method="ffill", inplace=True) + df["low"].fillna(method="ffill", inplace=True) + df["close"].fillna(method="ffill", inplace=True) + df["volume"].fillna(0, inplace=True) + df["market"] = market + df["granularity"] = granularity.to_integer + df["date"] = df.index + # reset pandas dataframe index df.reset_index() From 7222d7cb3c016d0629168ae420061ea2a68d6d85 Mon Sep 17 00:00:00 2001 From: Michael Whittle Date: Wed, 16 Aug 2023 16:45:23 +0100 Subject: [PATCH 2/2] added additional error handling --- models/config/binance_parser.py | 24 ++++++++++++++++-------- models/config/coinbase_parser.py | 24 ++++++++++++++++-------- models/config/coinbase_pro_parser.py | 28 ++++++++++++++++++---------- models/config/kucoin_parser.py | 27 +++++++++++++++++---------- 4 files changed, 67 insertions(+), 36 deletions(-) diff --git a/models/config/binance_parser.py b/models/config/binance_parser.py index 7214176e..a0fa89d1 100644 --- a/models/config/binance_parser.py +++ b/models/config/binance_parser.py @@ -2,6 +2,7 @@ import json import os.path import re +import sys from .default_parser import is_currency_valid, default_config_parse, merge_config_and_args @@ -101,14 +102,21 @@ def parser(app, binance_config, args={}): app.api_key_file = binance_config["api_key_file"] if app.api_key_file is not None: - try: - with open(app.api_key_file, "r") as f: - key = f.readline().strip() - secret = f.readline().strip() - binance_config["api_key"] = key - binance_config["api_secret"] = secret - except Exception: - raise RuntimeError(f"Unable to read {app.api_key_file}") + if not os.path.isfile(app.api_key_file): + try: + raise Exception(f"Unable to read {app.api_key_file}, please check the file exists and is readable. Remove it from the config file for test mode!\n") + except Exception as e: + print(f"{type(e).__name__}: {e}") + sys.exit(1) + else: + try: + with open(app.api_key_file, "r") as f: + key = f.readline().strip() + secret = f.readline().strip() + binance_config["api_key"] = key + binance_config["api_secret"] = secret + except Exception: + raise RuntimeError(f"Unable to read {app.api_key_file}") if "api_key" in binance_config and "api_secret" in binance_config and "api_url" in binance_config: # validates the api key is syntactically correct diff --git a/models/config/coinbase_parser.py b/models/config/coinbase_parser.py index 57e34914..d9871e26 100644 --- a/models/config/coinbase_parser.py +++ b/models/config/coinbase_parser.py @@ -2,6 +2,7 @@ import json import os.path import re +import sys from .default_parser import is_currency_valid, default_config_parse, merge_config_and_args from models.exchange.Granularity import Granularity @@ -56,14 +57,21 @@ def parser(app, coinbase_config, args={}): app.api_key_file = coinbase_config["api_key_file"] if app.api_key_file is not None: - try: - with open(app.api_key_file, "r") as f: - key = f.readline().strip() - secret = f.readline().strip() - coinbase_config["api_key"] = key - coinbase_config["api_secret"] = secret - except Exception: - raise RuntimeError(f"Unable to read {app.api_key_file}") + if not os.path.isfile(app.api_key_file): + try: + raise Exception(f"Unable to read {app.api_key_file}, please check the file exists and is readable. Remove \"api_key_file\" key from the config file for test mode!\n") + except Exception as e: + print(f"{type(e).__name__}: {e}") + sys.exit(1) + else: + try: + with open(app.api_key_file, "r") as f: + key = f.readline().strip() + secret = f.readline().strip() + coinbase_config["api_key"] = key + coinbase_config["api_secret"] = secret + except Exception: + raise RuntimeError(f"Unable to read {app.api_key_file}") if "api_url" in coinbase_config["config"]: coinbase_config["api_url"] = coinbase_config["config"]["api_url"] diff --git a/models/config/coinbase_pro_parser.py b/models/config/coinbase_pro_parser.py index ce1f1234..e2459377 100644 --- a/models/config/coinbase_pro_parser.py +++ b/models/config/coinbase_pro_parser.py @@ -2,6 +2,7 @@ import json import os.path import re +import sys from .default_parser import is_currency_valid, default_config_parse, merge_config_and_args from models.exchange.Granularity import Granularity @@ -57,16 +58,23 @@ def parser(app, coinbase_config, args={}): app.api_key_file = coinbase_config["api_key_file"] if app.api_key_file is not None: - try: - with open(app.api_key_file, "r") as f: - key = f.readline().strip() - secret = f.readline().strip() - password = f.readline().strip() - coinbase_config["api_key"] = key - coinbase_config["api_secret"] = secret - coinbase_config["api_passphrase"] = password - except Exception: - raise RuntimeError(f"Unable to read {app.api_key_file}") + if not os.path.isfile(app.api_key_file): + try: + raise Exception(f"Unable to read {app.api_key_file}, please check the file exists and is readable. Remove \"api_key_file\" key from the config file for test mode!\n") + except Exception as e: + print(f"{type(e).__name__}: {e}") + sys.exit(1) + else: + try: + with open(app.api_key_file, "r") as f: + key = f.readline().strip() + secret = f.readline().strip() + password = f.readline().strip() + coinbase_config["api_key"] = key + coinbase_config["api_secret"] = secret + coinbase_config["api_passphrase"] = password + except Exception: + raise RuntimeError(f"Unable to read {app.api_key_file}") if "api_key" in coinbase_config and "api_secret" in coinbase_config and "api_passphrase" in coinbase_config and "api_url" in coinbase_config: diff --git a/models/config/kucoin_parser.py b/models/config/kucoin_parser.py index 246e66eb..cdff1777 100644 --- a/models/config/kucoin_parser.py +++ b/models/config/kucoin_parser.py @@ -60,16 +60,23 @@ def parser(app, kucoin_config, args={}): app.api_key_file = kucoin_config["api_key_file"] if app.api_key_file is not None: - try: - with open(app.api_key_file, "r") as f: - key = f.readline().strip() - secret = f.readline().strip() - password = f.readline().strip() - kucoin_config["api_key"] = key - kucoin_config["api_secret"] = secret - kucoin_config["api_passphrase"] = password - except Exception: - raise RuntimeError(f"Unable to read {app.api_key_file}") + if not os.path.isfile(app.api_key_file): + try: + raise Exception(f"Unable to read {app.api_key_file}, please check the file exists and is readable. Remove \"api_key_file\" key from the config file for test mode!\n") + except Exception as e: + print(f"{type(e).__name__}: {e}") + sys.exit(1) + else: + try: + with open(app.api_key_file, "r") as f: + key = f.readline().strip() + secret = f.readline().strip() + password = f.readline().strip() + kucoin_config["api_key"] = key + kucoin_config["api_secret"] = secret + kucoin_config["api_passphrase"] = password + except Exception: + raise RuntimeError(f"Unable to read {app.api_key_file}") if "api_key" in kucoin_config and "api_secret" in kucoin_config and "api_passphrase" in kucoin_config and "api_url" in kucoin_config: # validates the api key is syntactically correct