Skip to content

Commit

Permalink
use aria2 instead of curl for download cell
Browse files Browse the repository at this point in the history
  • Loading branch information
Yoinky3000 committed Jun 29, 2024
1 parent bcd2f5e commit 2bfe239
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 6 deletions.
3 changes: 2 additions & 1 deletion assets/build/setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ apt install -y git \
python3.10-venv \
python3-pip \
curl \
wget > /dev/null
wget \
aria2 > /dev/null

echo Installing necessary python packages...
pip install jupyterlab "huggingface_hub[hf_transfer]" humanize > /dev/null
20 changes: 15 additions & 5 deletions assets/workspace/SD Next.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@
" \n",
" print(f'\\n\\nFetching \"{baseLink}\"...')\n",
" try:\n",
" response = requests.get(baseLink, allow_redirects=False, timeout=5, stream=True)\n",
" response = requests.get(baseLink, allow_redirects=True, timeout=10, stream=True)\n",
" finalLink = response.url\n",
" code = response.status_code \n",
" if code >= 400:\n",
Expand All @@ -119,13 +119,23 @@
" continue\n",
" \n",
" print(f'The link is the source link' if finalLink == baseLink else f'Resolved source link - \"{finalLink}\"')\n",
"\n",
" filename = filename if \".\" in filename else os.path.basename(urlsplit(finalLink).path)\n",
" \n",
" dest = f'\"{dl[1] if len(dl) == 2 else \"./\"}{filename}\"'\n",
" dest = dl[1] if len(dl) == 2 else \"./\"\n",
" resultPath = os.path.join(dest, filename)\n",
" dest = f'\"{dl[1] if len(dl) == 2 else \"./\"}\"'\n",
" filename = f'\"{filename}\"'\n",
" baseLink = f'\"{baseLink}\"'\n",
"\n",
" print(f'Downloading {filename} to {dest}...')\n",
" !curl --create-dirs -L $baseLink -o $dest\n",
" print(f'Downloading {filename} to {resultPath}...')\n",
" start = time.time()\n",
" !aria2c -x 16 -s 8 -d $dest -o $filename $baseLink > /dev/null\n",
" end = time.time()\n",
" timeUsed = end - start\n",
" fileSize = os.path.getsize(resultPath)\n",
" avgSpeed = humanize.naturalsize(fileSize/timeUsed, binary=True)\n",
" fileSize = humanize.naturalsize(fileSize, binary=True)\n",
" print(f'Download completed\\n- Time used: {timeUsed}s\\n- File size: {fileSize}\\n- Average speed: {avgSpeed}/s')\n",
"\n",
"print(\"\\n\\nDownload finished\")"
]
Expand Down

0 comments on commit 2bfe239

Please sign in to comment.