From 633b3ca6ae8637634952217441f5834e02e233fb Mon Sep 17 00:00:00 2001 From: Patrick Li Date: Fri, 13 Sep 2024 15:57:47 -0700 Subject: [PATCH] adding some debugging logs --- ultravox/tools/ds_tool/ds_tool.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ultravox/tools/ds_tool/ds_tool.py b/ultravox/tools/ds_tool/ds_tool.py index 6feaa126..4ecaebef 100644 --- a/ultravox/tools/ds_tool/ds_tool.py +++ b/ultravox/tools/ds_tool/ds_tool.py @@ -446,7 +446,11 @@ def _upload(self, ds_chunk_processed: datasets.Dataset, data_dir: str, split_nam "split": split_name, } assert isinstance(self.args.upload_name, str) - ds_split_chunked.push_to_hub(self.args.upload_name, **hub_args) + try: + ds_split_chunked.push_to_hub(self.args.upload_name, **hub_args) + except Exception as e: + print(f"Failed to upload chunk to hub: {e}") + raise e def main(args: DatasetToolArgs): @@ -478,8 +482,6 @@ def main(args: DatasetToolArgs): if args.num_samples: ds_split = ds_split.select(range(args.num_samples)) - if args.upload_split: - split_name = args.upload_split ds_chunk_proc.process_and_upload_split_rescursive( split_name, ds_split, 0, len(ds_split) )