From 15281822f876ed6092e8b5be7010d79e3ae1ebf4 Mon Sep 17 00:00:00 2001 From: photino Date: Mon, 3 Jul 2023 09:03:59 +0800 Subject: [PATCH] Add support for CSV exporter --- examples/actix-app/src/router/mod.rs | 4 +- zino-core/Cargo.toml | 7 +-- zino-core/src/extension/json_value.rs | 68 ++++++++++++++++++++------- zino-core/src/response/mod.rs | 42 +++++++++-------- zino/Cargo.toml | 2 +- zino/src/controller/mod.rs | 30 ++++++++++-- 6 files changed, 107 insertions(+), 46 deletions(-) diff --git a/examples/actix-app/src/router/mod.rs b/examples/actix-app/src/router/mod.rs index 2fcd7651..2d32b386 100644 --- a/examples/actix-app/src/router/mod.rs +++ b/examples/actix-app/src/router/mod.rs @@ -17,7 +17,9 @@ fn user_router(cfg: &mut ServiceConfig) { .route("/user/{id}/delete", post().to(User::delete)) .route("/user/{id}/update", post().to(User::update)) .route("/user/{id}/view", get().to(user::view)) - .route("/user/list", get().to(User::list)); + .route("/user/list", get().to(User::list)) + .route("/user/import", post().to(User::import)) + .route("/user/export", get().to(User::export)); } fn tag_router(cfg: &mut ServiceConfig) { diff --git a/zino-core/Cargo.toml b/zino-core/Cargo.toml index 048fd0b1..779cca9a 100644 --- a/zino-core/Cargo.toml +++ b/zino-core/Cargo.toml @@ -83,6 +83,7 @@ cfg-if = "1.0" convert_case = "0.6.0" cookie = "0.16.2" cron = "0.12.0" +csv = "1.2.2" fluent = "0.16.0" futures = "0.3.28" hkdf = "0.12.3" @@ -103,7 +104,7 @@ rmp-serde = "1.1.1" serde_qs = "0.12.0" serde_urlencoded = "0.7.1" sha2 = "0.10.7" -sysinfo = "0.29.2" +sysinfo = "0.29.3" task-local-extensions = "0.1.4" toml = "0.7.5" tracing = "0.1.37" @@ -123,11 +124,11 @@ version = "0.4.26" features = ["serde"] [dependencies.datafusion] -version = "26.0.0" +version = "27.0.0" optional = true [dependencies.lru] -version = "0.10.0" +version = "0.10.1" optional = true [dependencies.minijinja] diff --git a/zino-core/src/extension/json_value.rs b/zino-core/src/extension/json_value.rs index 4b4f965d..6f5ea970 100644 --- a/zino-core/src/extension/json_value.rs +++ b/zino-core/src/extension/json_value.rs @@ -1,6 +1,8 @@ -use crate::JsonValue; +use crate::{extension::JsonObjectExt, JsonValue}; +use csv::{ByteRecord, Writer}; use std::{ borrow::Cow, + io::{self, ErrorKind, Write}, num::{ParseFloatError, ParseIntError}, str::{FromStr, ParseBoolError}, }; @@ -31,47 +33,50 @@ pub trait JsonValueExt { /// Returns `None` otherwise. fn as_f32(&self) -> Option; - /// Parses the json value as `bool`. + /// Parses the JSON value as `bool`. fn parse_bool(&self) -> Option>; - /// Parses the json value as `u8`. + /// Parses the JSON value as `u8`. fn parse_u8(&self) -> Option>; - /// Parses the json value as `u16`. + /// Parses the JSON value as `u16`. fn parse_u16(&self) -> Option>; - /// Parses the json value as `u32`. + /// Parses the JSON value as `u32`. fn parse_u32(&self) -> Option>; - /// Parses the json value as `u64`. + /// Parses the JSON value as `u64`. fn parse_u64(&self) -> Option>; - /// Parses the json value as `usize`. + /// Parses the JSON value as `usize`. fn parse_usize(&self) -> Option>; - /// Parses the json value as `i32`. + /// Parses the JSON value as `i32`. fn parse_i32(&self) -> Option>; - /// Parses the json value as `i64`. + /// Parses the JSON value as `i64`. fn parse_i64(&self) -> Option>; - /// Parses the json value as `f32`. + /// Parses the JSON value as `f32`. fn parse_f32(&self) -> Option>; - /// Parses the json value as `f64`. + /// Parses the JSON value as `f64`. fn parse_f64(&self) -> Option>; - /// Parses the json value as `Cow<'_, str>`. + /// Parses the JSON value as `Cow<'_, str>`. /// If the str is empty, it also returns `None`. fn parse_string(&self) -> Option>; - /// Parses the json value as `Vec`. + /// Parses the JSON value as `Vec`. /// If the vec is empty, it also returns `None`. fn parse_array(&self) -> Option>; - /// Parses the json value as `Vec<&str>`. + /// Parses the JSON value as `Vec<&str>`. /// If the vec is empty, it also returns `None`. fn parse_str_array(&self) -> Option>; + + /// Attempts to convert the JSON value to a CSV writer. + fn to_csv_writer(&self, writer: W) -> Result; } impl JsonValueExt for JsonValue { @@ -185,7 +190,7 @@ impl JsonValueExt for JsonValue { fn parse_array(&self) -> Option> { let values = match &self { JsonValue::String(s) => Some(crate::format::parse_str_array(s)), - JsonValue::Array(v) => Some(v.iter().filter_map(|v| v.as_str()).collect()), + JsonValue::Array(vec) => Some(vec.iter().filter_map(|v| v.as_str()).collect()), _ => None, }; let vec = values? @@ -198,7 +203,7 @@ impl JsonValueExt for JsonValue { fn parse_str_array(&self) -> Option> { let values = match &self { JsonValue::String(s) => Some(crate::format::parse_str_array(s)), - JsonValue::Array(v) => Some(v.iter().filter_map(|v| v.as_str()).collect()), + JsonValue::Array(vec) => Some(vec.iter().filter_map(|v| v.as_str()).collect()), _ => None, }; let vec = values? @@ -208,4 +213,35 @@ impl JsonValueExt for JsonValue { .collect::>(); (!vec.is_empty()).then_some(vec) } + + fn to_csv_writer(&self, writer: W) -> Result { + match &self { + JsonValue::Array(vec) => { + let mut wtr = Writer::from_writer(writer); + let mut headers = Vec::new(); + if let Some(JsonValue::Object(map)) = vec.first() { + for key in map.keys() { + headers.push(key.to_owned()); + } + } + wtr.write_record(&headers)?; + + let num_fields = headers.len(); + let buffer_size = num_fields * 8; + for value in vec { + if let JsonValue::Object(map) = value { + let mut record = ByteRecord::with_capacity(buffer_size, num_fields); + for field in headers.iter() { + let value = map.parse_string(field).unwrap_or("".into()); + record.push_field(value.as_ref().as_bytes()); + } + wtr.write_byte_record(&record)?; + } + } + wtr.flush()?; + wtr.into_inner().map_err(|err| err.into_error().into()) + } + _ => Err(io::Error::new(ErrorKind::InvalidData, "invalid JSON value for CSV").into()), + } + } } diff --git a/zino-core/src/response/mod.rs b/zino-core/src/response/mod.rs index 8930c48c..261d779c 100644 --- a/zino-core/src/response/mod.rs +++ b/zino-core/src/response/mod.rs @@ -29,6 +29,9 @@ pub type StatusCode = http::StatusCode; /// An Http response with the body that consists of a single chunk. pub type FullResponse = http::Response>; +/// A function pointer of transforming the response data. +pub type ResponseDataTransformer = fn(data: JsonValue) -> Result, Error>; + /// An HTTP response. #[derive(Debug, Serialize)] #[serde(rename_all = "snake_case")] @@ -67,9 +70,9 @@ pub struct Response { /// Response data. #[serde(skip_serializing_if = "Option::is_none")] data: Option>, - /// JSON Pointer for identifying a specific value within the response data. + /// Transformer of the response data. #[serde(skip)] - json_pointer: Option, + data_transformer: Option, /// Content type. #[serde(skip)] content_type: Option, @@ -104,7 +107,7 @@ impl Response { start_time: Instant::now(), request_id: Uuid::nil(), data: None, - json_pointer: None, + data_transformer: None, content_type: None, trace_context: None, server_timing: ServerTiming::new(), @@ -135,7 +138,7 @@ impl Response { start_time: ctx.start_time(), request_id: ctx.request_id(), data: None, - json_pointer: ctx.get_query("json_pointer").map(|s| s.to_owned().into()), + data_transformer: None, content_type: None, trace_context: None, server_timing: ServerTiming::new(), @@ -157,7 +160,6 @@ impl Response { self.start_time = ctx.start_time(); self.request_id = ctx.request_id(); self.trace_context = Some(ctx.new_trace_context()); - self.json_pointer = ctx.get_query("json_pointer").map(|s| s.to_owned().into()); self } @@ -264,10 +266,10 @@ impl Response { } } - /// Sets a JSON Pointer for identifying a specific value within the response data. + /// Sets a transformer for the response data. #[inline] - pub fn set_json_pointer(&mut self, pointer: impl Into) { - self.json_pointer = Some(pointer.into()); + pub fn set_data_transformer(&mut self, transformer: ResponseDataTransformer) { + self.data_transformer = Some(transformer); } /// Sets the content type. @@ -414,21 +416,21 @@ impl Response { /// Reads the response into a byte buffer. pub fn read_bytes(&self) -> Result, Error> { + if let Some(transformer) = self.data_transformer.as_ref() { + let data = serde_json::to_value(&self.data)?; + return transformer(data); + } + let content_type = self.content_type(); let bytes = if extension::header::check_json_content_type(content_type) { - if let Some(pointer) = self.json_pointer.as_deref() { - let data = serde_json::to_value(&self.data)?; - serde_json::to_vec(&data.pointer(pointer))? + let capacity = if let Some(data) = &self.data { + data.get().len() + 128 } else { - let capacity = if let Some(data) = &self.data { - data.get().len() + 128 - } else { - 128 - }; - let mut bytes = Vec::with_capacity(capacity); - serde_json::to_writer(&mut bytes, &self)?; - bytes - } + 128 + }; + let mut bytes = Vec::with_capacity(capacity); + serde_json::to_writer(&mut bytes, &self)?; + bytes } else if let Some(data) = &self.data { let capacity = data.get().len(); match serde_json::to_value(data)? { diff --git a/zino/Cargo.toml b/zino/Cargo.toml index cbfe0779..67a1a77e 100644 --- a/zino/Cargo.toml +++ b/zino/Cargo.toml @@ -90,7 +90,7 @@ version = "3.1.3" optional = true [dependencies.tokio] -version = "1.29.0" +version = "1.29.1" optional = true features = ["parking_lot", "rt-multi-thread"] diff --git a/zino/src/controller/mod.rs b/zino/src/controller/mod.rs index 57eb0bf2..844f6b27 100644 --- a/zino/src/controller/mod.rs +++ b/zino/src/controller/mod.rs @@ -31,9 +31,10 @@ pub trait DefaultController { #[cfg(feature = "orm")] use zino_core::{ database::ModelAccessor, + error::Error, extension::JsonObjectExt, request::RequestContext, - response::{ExtractRejection, Rejection}, + response::{ExtractRejection, Rejection, StatusCode}, Map, }; @@ -57,7 +58,7 @@ where let data = Map::data_entry(model.snapshot()); model.insert().await.extract(&req)?; - res.set_code(zino_core::response::StatusCode::CREATED); + res.set_code(StatusCode::CREATED); res.set_data(&data); Ok(res.into()) } @@ -111,8 +112,6 @@ where } async fn import(mut req: Self::Request) -> Self::Result { - use zino_core::response::StatusCode; - let data = req.parse_body::>().await?; let mut models = Vec::with_capacity(data.len()); let mut validations = Vec::new(); @@ -150,7 +149,28 @@ where let models = Self::find(&query).await.extract(&req)?; let data = Map::data_entries(models); res.set_data(&data); - res.set_json_pointer("/entries"); + + let format = req.get_query("format").unwrap_or("json"); + match format { + "csv" => { + use zino_core::extension::JsonValueExt; + + res.set_content_type("text/csv; charset=utf-8"); + res.set_data_transformer(|data| { + let bytes = if let Some(value) = data.pointer("/entries") { + value.to_csv_writer(Vec::new())? + } else { + Vec::new() + }; + Ok(bytes) + }); + } + _ => { + res.set_data_transformer(|data| { + serde_json::to_vec(&data.pointer("/entries")).map_err(Error::from) + }); + } + } Ok(res.into()) } }