From c1e17ad22743c0e407ee03d66396af8fbbe98052 Mon Sep 17 00:00:00 2001 From: Alexander Sysoev Date: Fri, 23 Sep 2022 19:30:08 +0300 Subject: [PATCH] Refactor ipfs upload, add infura auth (#59) --- README.md | 17 +++++++++------ stakewise_cli/ipfs.py | 45 ++++++++++++++++++++++++++++----------- stakewise_cli/settings.py | 23 ++++++++++++++------ 3 files changed, 58 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 98218a5..57e3d51 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,13 @@ Your validators will get ether assigned, and you can claim your operator rewards ### Operator CLI Environment Settings -| Variable | Description | Required | Default | -|--------------------------------|----------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------| -| IPFS_PIN_ENDPOINTS | The IPFS endpoint where the deposit data will be uploaded | No | /dns/ipfs.infura.io/tcp/5001/https | -| IPFS_FETCH_ENDPOINTS | The IPFS endpoints from where the deposit data will be fetched | No | https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io | -| IPFS_PINATA_API_KEY | The Pinata API key for uploading deposit data for the redundancy | No | - | -| IPFS_PINATA_SECRET_KEY | The Pinata Secret key for uploading deposit data for the redundancy | No | - | -| VAULT_VALIDATORS_MOUNT_POINT | The mount point in Hashicorp Vault for storing validator keys | No | validators | +| Variable | Description | Required | Default | +|--------------------------------|----------------------------------------------------------------------------|----------|-------------------------------------------------------------------------| +| INFURA_IPFS_CLIENT_ENDPOINT | The http://infura.io IPFS endpoint where the deposit data will be uploaded | No | /dns/ipfs.infura.io/tcp/5001/https | +| INFURA_IPFS_CLIENT_USERNAME | The http://infura.io IPFS account username | No | - | +| INFURA_IPFS_CLIENT_PASSWORD | The http://infura.io IPFS account password | No | - | +| LOCAL_IPFS_CLIENT_ENDPOINT | The local IPFS endpoints from where the deposit data will be uploaded | No | - | +| IPFS_EXTRA_FETCH_ENDPOINTS | The extra IPFS endpoints from where the deposit data will be fetched | No | https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io | +| IPFS_PINATA_API_KEY | The Pinata API key for uploading deposit data for the redundancy | No | - | +| IPFS_PINATA_SECRET_KEY | The Pinata Secret key for uploading deposit data for the redundancy | No | - | +| VAULT_VALIDATORS_MOUNT_POINT | The mount point in Hashicorp Vault for storing validator keys | No | validators | diff --git a/stakewise_cli/ipfs.py b/stakewise_cli/ipfs.py index 6c9f1e0..5f276f9 100644 --- a/stakewise_cli/ipfs.py +++ b/stakewise_cli/ipfs.py @@ -7,11 +7,14 @@ import requests from stakewise_cli.settings import ( - IPFS_FETCH_ENDPOINTS, - IPFS_PIN_ENDPOINTS, + INFURA_IPFS_CLIENT_ENDPOINT, + INFURA_IPFS_CLIENT_PASSWORD, + INFURA_IPFS_CLIENT_USERNAME, + IPFS_EXTRA_FETCH_ENDPOINTS, IPFS_PINATA_API_KEY, IPFS_PINATA_PIN_ENDPOINT, IPFS_PINATA_SECRET_KEY, + LOCAL_IPFS_CLIENT_ENDPOINT, ) @@ -29,15 +32,28 @@ def add_ipfs_prefix(ipfs_id: str) -> str: def upload_to_ipfs(data: Any) -> str: """Submits data to IPFS.""" ipfs_ids = [] - for pin_endpoint in IPFS_PIN_ENDPOINTS: + try: + with ipfshttpclient.connect( + INFURA_IPFS_CLIENT_ENDPOINT, + username=INFURA_IPFS_CLIENT_USERNAME, + password=INFURA_IPFS_CLIENT_PASSWORD, + ) as client: + ipfs_id = client.add_json(data) + client.pin.add(ipfs_id) + ipfs_ids.append(ipfs_id) + except Exception as e: + click.echo(e) + click.echo(f"Failed to submit data to {INFURA_IPFS_CLIENT_ENDPOINT}") + + if LOCAL_IPFS_CLIENT_ENDPOINT: try: - with ipfshttpclient.connect(pin_endpoint) as client: + with ipfshttpclient.connect(LOCAL_IPFS_CLIENT_ENDPOINT) as client: ipfs_id = client.add_json(data) client.pin.add(ipfs_id) ipfs_ids.append(ipfs_id) except Exception as e: click.echo(e) - click.echo(f"Failed to submit data to {pin_endpoint}") + click.echo(f"Failed to submit data to {LOCAL_IPFS_CLIENT_ENDPOINT}") if IPFS_PINATA_API_KEY and IPFS_PINATA_SECRET_KEY: headers = { @@ -72,14 +88,17 @@ def upload_to_ipfs(data: Any) -> str: def ipfs_fetch(ipfs_id: str) -> Any: """Fetches data from IPFS.""" ipfs_id = ipfs_id.replace("ipfs://", "").replace("/ipfs/", "") - for ipfs_endpoint in IPFS_PIN_ENDPOINTS: - try: - with ipfshttpclient.connect(ipfs_endpoint) as client: - return client.get_json(ipfs_id) - except: # noqa: E722 - pass - - for endpoint in IPFS_FETCH_ENDPOINTS: + try: + with ipfshttpclient.connect( + INFURA_IPFS_CLIENT_ENDPOINT, + username=INFURA_IPFS_CLIENT_USERNAME, + password=INFURA_IPFS_CLIENT_PASSWORD, + ) as client: + return client.get_json(ipfs_id) + except: # noqa: E722 + pass + + for endpoint in IPFS_EXTRA_FETCH_ENDPOINTS: try: response = requests.get(f"{endpoint.rstrip('/')}/ipfs/{ipfs_id}") response.raise_for_status() diff --git a/stakewise_cli/settings.py b/stakewise_cli/settings.py index e9518e2..a2f5d5d 100644 --- a/stakewise_cli/settings.py +++ b/stakewise_cli/settings.py @@ -1,14 +1,17 @@ from decouple import Csv, config # extra pins to pinata for redundancy -IPFS_PIN_ENDPOINTS = config( - "IPFS_PIN_ENDPOINTS", cast=Csv(), default="/dns/ipfs.infura.io/tcp/5001/https" -) -IPFS_FETCH_ENDPOINTS = config( - "IPFS_FETCH_ENDPOINTS", - cast=Csv(), - default="https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io", +LOCAL_IPFS_CLIENT_ENDPOINT = config("LOCAL_IPFS_CLIENT_ENDPOINT", default="") + +# infura +INFURA_IPFS_CLIENT_ENDPOINT = config( + "INFURA_IPFS_CLIENT_ENDPOINT", + default="/dns/ipfs.infura.io/tcp/5001/https", ) +INFURA_IPFS_CLIENT_USERNAME = config("INFURA_IPFS_CLIENT_USERNAME", default="") +INFURA_IPFS_CLIENT_PASSWORD = config("INFURA_IPFS_CLIENT_PASSWORD", default="") + +# pinata IPFS_PINATA_PIN_ENDPOINT = config( "IPFS_PINATA_ENDPOINT", default="https://api.pinata.cloud/pinning/pinJSONToIPFS" ) @@ -18,6 +21,12 @@ default="", ) +IPFS_EXTRA_FETCH_ENDPOINTS = config( + "IPFS_FETCH_ENDPOINTS", + cast=Csv(), + default="https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io", +) + VAULT_VALIDATORS_MOUNT_POINT = config( "VAULT_VALIDATORS_MOUNT_POINT", default="validators" )