Skip to content

Commit

Permalink
downloader was double downloading
Browse files Browse the repository at this point in the history
  • Loading branch information
dan-garvey committed Jun 23, 2023
1 parent cdd505e commit 1b63989
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions shark/shark_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,15 @@ def download_public_file(
else:
continue

destination_filename = os.path.join(destination_folder_name, blob_name)
if os.path.isdir(destination_filename):
continue
with open(destination_filename, "wb") as f:
with tqdm.wrapattr(f, "write", total=blob.size) as file_obj:
storage_client.download_blob_to_file(blob, file_obj)
else:
destination_filename = os.path.join(
destination_folder_name, blob_name
)
if os.path.isdir(destination_filename):
continue
with open(destination_filename, "wb") as f:
with tqdm.wrapattr(f, "write", total=blob.size) as file_obj:
storage_client.download_blob_to_file(blob, file_obj)


input_type_to_np_dtype = {
Expand Down

0 comments on commit 1b63989

Please sign in to comment.