From c67239b09eb4511932d5702bb2ffa41134bf35c9 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 Dec 2024 17:11:10 +0800 Subject: [PATCH 1/2] feat: Use opendal http instead of hand written web-sys request Signed-off-by: Xuanwo --- Cargo.toml | 2 +- src/file_reader.rs | 83 +++++++++++++++------------------------------- 2 files changed, 27 insertions(+), 58 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fcd5b93..7e9e9be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ zstd = { version = "*", features = ["wasm", "thin"], default-features = false } zstd-sys = { version = "=2.0.9", default-features = false } serde = { version = "1.0", features = ["derive"] } opendal = { version = "0.50.2", default-features = false, features = [ - "services-s3", + "services-s3", "services-http" ] } [profile.release] diff --git a/src/file_reader.rs b/src/file_reader.rs index ae4f612..eefa77a 100644 --- a/src/file_reader.rs +++ b/src/file_reader.rs @@ -1,9 +1,8 @@ use bytes::Bytes; use leptos::prelude::*; use leptos::wasm_bindgen::{prelude::Closure, JsCast}; -use opendal::{services::S3, Operator}; -use wasm_bindgen_futures::JsFuture; -use web_sys::js_sys; +use opendal::{services::Http, services::S3, Operator}; +use web_sys::{js_sys, Url}; const S3_ENDPOINT_KEY: &str = "s3_endpoint"; const S3_ACCESS_KEY_ID_KEY: &str = "s3_access_key_id"; @@ -29,53 +28,6 @@ fn save_to_storage(key: &str, value: &str) { } } -async fn fetch_parquet_from_url(url_str: String) -> Result { - let opts = web_sys::RequestInit::new(); - opts.set_method("GET"); - - let headers = web_sys::Headers::new().map_err(|_| "Failed to create headers")?; - headers - .append("Accept", "*/*") - .map_err(|_| "Failed to set headers")?; - opts.set_headers(&headers); - - let request = web_sys::Request::new_with_str_and_init(&url_str, &opts) - .map_err(|_| "Failed to create request")?; - - let window = web_sys::window().ok_or("Failed to get window")?; - let resp = JsFuture::from(window.fetch_with_request(&request)) - .await - .map_err(|_| "Failed to fetch the file. This might be due to CORS restrictions. Try using a direct link from S3 or a server that allows CORS.")?; - - let resp: web_sys::Response = resp.dyn_into().map_err(|_| "Failed to convert response")?; - - if !resp.ok() { - let status = resp.status(); - let error_msg = match status { - 0 => "Network error: The server might be blocking CORS requests.", - 403 => "Access denied: The file is not publicly accessible.", - 404 => "File not found: Please check if the URL is correct.", - _ => { - return Err(format!( - "Server error (status {}): Please try a different source.", - status - )) - } - }; - return Err(error_msg.to_string()); - } - - let array_buffer = JsFuture::from( - resp.array_buffer() - .map_err(|_| "Failed to get array buffer")?, - ) - .await - .map_err(|_| "Failed to convert array buffer")?; - - let uint8_array = js_sys::Uint8Array::new(&array_buffer); - Ok(Bytes::from(uint8_array.to_vec())) -} - #[component] pub fn FileReader( set_error_message: WriteSignal>, @@ -139,7 +91,16 @@ pub fn FileReader( let url_str = url.get(); set_error_message.set(None); - let table_name = url_str + let Ok(url) = Url::new(&url_str) else { + set_error_message.set(Some(format!("Invalid URL: {}", url_str))); + return; + }; + // NOTE: protocol will include `:`, for example: `https:` + let endpoint = format!("{}//{}", url.protocol(), url.host()); + // We don't support query so far. + let path = url.pathname(); + + let table_name = path .split('/') .last() .unwrap_or("uploaded.parquet") @@ -149,12 +110,21 @@ pub fn FileReader( set_file_name.set(table_name); wasm_bindgen_futures::spawn_local(async move { - match fetch_parquet_from_url(url_str).await { - Ok(bytes) => { - set_file_bytes.set(Some(bytes.clone())); + let builder = Http::default().endpoint(&endpoint); + let Ok(op) = Operator::new(builder) else { + set_error_message.set(Some("Failed to create HTTP operator".into())); + return; + }; + let op = op.finish(); + + match op.read(&path).await { + Ok(bs) => { + set_file_bytes.set(Some(bs.to_bytes())); set_is_folded.set(true); } - Err(error) => set_error_message.set(Some(error)), + Err(e) => { + set_error_message.set(Some(format!("Failed to read from S3: {}", e))); + } } }); }; @@ -197,8 +167,7 @@ pub fn FileReader( let operator = op.finish(); match operator.read(&s3_file_path.get()).await { Ok(bs) => { - let bytes = Bytes::from(bs.to_vec()); - set_file_bytes.set(Some(bytes.clone())); + set_file_bytes.set(Some(bs.to_bytes())); set_is_folded.set(true); } Err(e) => { From ecd9bf1cce3052911e0dcad42f90300d07256cb6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 6 Dec 2024 17:14:35 +0800 Subject: [PATCH 2/2] Fix error message Signed-off-by: Xuanwo --- src/file_reader.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/file_reader.rs b/src/file_reader.rs index eefa77a..07475fe 100644 --- a/src/file_reader.rs +++ b/src/file_reader.rs @@ -123,7 +123,7 @@ pub fn FileReader( set_is_folded.set(true); } Err(e) => { - set_error_message.set(Some(format!("Failed to read from S3: {}", e))); + set_error_message.set(Some(format!("Failed to read from HTTP: {}", e))); } } });