Refactor ipfs upload, add infura auth (#59)

stakewise · Sep 23, 2022 · c1e17ad · c1e17ad
1 parent ac25fde
commit c1e17ad
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 27 deletions.
diff --git a/README.md b/README.md
@@ -64,10 +64,13 @@ Your validators will get ether assigned, and you can claim your operator rewards
 
 ### Operator CLI Environment Settings
 
-| Variable                       | Description                                                                      | Required | Default                                                                 |
-|--------------------------------|----------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------|
-| IPFS_PIN_ENDPOINTS             | The IPFS endpoint where the deposit data will be uploaded                        | No       | /dns/ipfs.infura.io/tcp/5001/https                                      |
-| IPFS_FETCH_ENDPOINTS           | The IPFS endpoints from where the deposit data will be fetched                   | No       | https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io |
-| IPFS_PINATA_API_KEY            | The Pinata API key for uploading deposit data for the redundancy                 | No       | -                                                                       |
-| IPFS_PINATA_SECRET_KEY         | The Pinata Secret key for uploading deposit data for the redundancy              | No       | -                                                                       |
-| VAULT_VALIDATORS_MOUNT_POINT   | The mount point in Hashicorp Vault for storing validator keys                    | No       | validators                                                              |
+| Variable                       | Description                                                                | Required | Default                                                                 |
+|--------------------------------|----------------------------------------------------------------------------|----------|-------------------------------------------------------------------------|
+| INFURA_IPFS_CLIENT_ENDPOINT    | The http://infura.io IPFS endpoint where the deposit data will be uploaded | No       | /dns/ipfs.infura.io/tcp/5001/https                                      |
+| INFURA_IPFS_CLIENT_USERNAME    | The http://infura.io IPFS account username                                 | No       | -                                                                       |
+| INFURA_IPFS_CLIENT_PASSWORD    | The http://infura.io IPFS account password                                 | No       | -                                                                       |
+| LOCAL_IPFS_CLIENT_ENDPOINT     | The local IPFS endpoints from where the deposit data will be uploaded      | No       | -                                                                       |
+| IPFS_EXTRA_FETCH_ENDPOINTS     | The extra IPFS endpoints from where the deposit data will be fetched       | No       | https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io |
+| IPFS_PINATA_API_KEY            | The Pinata API key for uploading deposit data for the redundancy           | No       | -                                                                       |
+| IPFS_PINATA_SECRET_KEY         | The Pinata Secret key for uploading deposit data for the redundancy        | No       | -                                                                       |
+| VAULT_VALIDATORS_MOUNT_POINT   | The mount point in Hashicorp Vault for storing validator keys              | No       | validators                                                              |
diff --git a/stakewise_cli/ipfs.py b/stakewise_cli/ipfs.py
@@ -7,11 +7,14 @@
 import requests
 
 from stakewise_cli.settings import (
-    IPFS_FETCH_ENDPOINTS,
-    IPFS_PIN_ENDPOINTS,
+    INFURA_IPFS_CLIENT_ENDPOINT,
+    INFURA_IPFS_CLIENT_PASSWORD,
+    INFURA_IPFS_CLIENT_USERNAME,
+    IPFS_EXTRA_FETCH_ENDPOINTS,
     IPFS_PINATA_API_KEY,
     IPFS_PINATA_PIN_ENDPOINT,
     IPFS_PINATA_SECRET_KEY,
+    LOCAL_IPFS_CLIENT_ENDPOINT,
 )
 
 
@@ -29,15 +32,28 @@ def add_ipfs_prefix(ipfs_id: str) -> str:
 def upload_to_ipfs(data: Any) -> str:
     """Submits data to IPFS."""
     ipfs_ids = []
-    for pin_endpoint in IPFS_PIN_ENDPOINTS:
+    try:
+        with ipfshttpclient.connect(
+            INFURA_IPFS_CLIENT_ENDPOINT,
+            username=INFURA_IPFS_CLIENT_USERNAME,
+            password=INFURA_IPFS_CLIENT_PASSWORD,
+        ) as client:
+            ipfs_id = client.add_json(data)
+            client.pin.add(ipfs_id)
+            ipfs_ids.append(ipfs_id)
+    except Exception as e:
+        click.echo(e)
+        click.echo(f"Failed to submit data to {INFURA_IPFS_CLIENT_ENDPOINT}")
+
+    if LOCAL_IPFS_CLIENT_ENDPOINT:
         try:
-            with ipfshttpclient.connect(pin_endpoint) as client:
+            with ipfshttpclient.connect(LOCAL_IPFS_CLIENT_ENDPOINT) as client:
                 ipfs_id = client.add_json(data)
                 client.pin.add(ipfs_id)
                 ipfs_ids.append(ipfs_id)
         except Exception as e:
             click.echo(e)
-            click.echo(f"Failed to submit data to {pin_endpoint}")
+            click.echo(f"Failed to submit data to {LOCAL_IPFS_CLIENT_ENDPOINT}")
 
     if IPFS_PINATA_API_KEY and IPFS_PINATA_SECRET_KEY:
         headers = {
@@ -72,14 +88,17 @@ def upload_to_ipfs(data: Any) -> str:
 def ipfs_fetch(ipfs_id: str) -> Any:
     """Fetches data from IPFS."""
     ipfs_id = ipfs_id.replace("ipfs://", "").replace("/ipfs/", "")
-    for ipfs_endpoint in IPFS_PIN_ENDPOINTS:
-        try:
-            with ipfshttpclient.connect(ipfs_endpoint) as client:
-                return client.get_json(ipfs_id)
-        except:  # noqa: E722
-            pass
-
-    for endpoint in IPFS_FETCH_ENDPOINTS:
+    try:
+        with ipfshttpclient.connect(
+            INFURA_IPFS_CLIENT_ENDPOINT,
+            username=INFURA_IPFS_CLIENT_USERNAME,
+            password=INFURA_IPFS_CLIENT_PASSWORD,
+        ) as client:
+            return client.get_json(ipfs_id)
+    except:  # noqa: E722
+        pass
+
+    for endpoint in IPFS_EXTRA_FETCH_ENDPOINTS:
         try:
             response = requests.get(f"{endpoint.rstrip('/')}/ipfs/{ipfs_id}")
             response.raise_for_status()

diff --git a/stakewise_cli/settings.py b/stakewise_cli/settings.py
@@ -1,14 +1,17 @@
 from decouple import Csv, config
 
 # extra pins to pinata for redundancy
-IPFS_PIN_ENDPOINTS = config(
-    "IPFS_PIN_ENDPOINTS", cast=Csv(), default="/dns/ipfs.infura.io/tcp/5001/https"
-)
-IPFS_FETCH_ENDPOINTS = config(
-    "IPFS_FETCH_ENDPOINTS",
-    cast=Csv(),
-    default="https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io",
+LOCAL_IPFS_CLIENT_ENDPOINT = config("LOCAL_IPFS_CLIENT_ENDPOINT", default="")
+
+# infura
+INFURA_IPFS_CLIENT_ENDPOINT = config(
+    "INFURA_IPFS_CLIENT_ENDPOINT",
+    default="/dns/ipfs.infura.io/tcp/5001/https",
 )
+INFURA_IPFS_CLIENT_USERNAME = config("INFURA_IPFS_CLIENT_USERNAME", default="")
+INFURA_IPFS_CLIENT_PASSWORD = config("INFURA_IPFS_CLIENT_PASSWORD", default="")
+
+# pinata
 IPFS_PINATA_PIN_ENDPOINT = config(
     "IPFS_PINATA_ENDPOINT", default="https://api.pinata.cloud/pinning/pinJSONToIPFS"
 )
@@ -18,6 +21,12 @@
     default="",
 )
 
+IPFS_EXTRA_FETCH_ENDPOINTS = config(
+    "IPFS_FETCH_ENDPOINTS",
+    cast=Csv(),
+    default="https://gateway.pinata.cloud,http://cloudflare-ipfs.com,https://ipfs.io",
+)
+
 VAULT_VALIDATORS_MOUNT_POINT = config(
     "VAULT_VALIDATORS_MOUNT_POINT", default="validators"
 )