feat(tls): enable tls authentication

- update client for better metadata headers
- add feature flag for tls to enable connection to Databricks
- update sparksession builder to accetp &str and not String for connection string

Cargo.toml

@@ -37,7 +37,10 @@ url = "2.2"
 tonic-build = "0.11.0"
 
 [lib]
+
 doctest = false
+[features]
+tls = ["tonic/tls", "tonic/tls-roots"]
 
 [[example]]
 name = "reader"
@@ -47,3 +50,10 @@ name = "sql"
 
 [[example]]
 name = "writer"
+
+[[example]]
+name = "delta"
+
+[[example]]
+name = "databricks"
+required-features = ["tls"]

README.md

@@ -28,7 +28,7 @@ use spark_connect_rs::functions as F;
 
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/".to_string())
+    let spark: SparkSession = SparkSessionBuilder::remote("sc://127.0.0.1:15002/")
             .build()
             .await?;
 
@@ -74,7 +74,7 @@ cargo build && cargo test
 The following section outlines some of the larger functionality that
 is not yet working with this Spark Connect implementation.
 
-- ![open] TLS authentication & Databricks compatability
+- ![done] TLS authentication & Databricks compatability
 - ![open] streaming implementation
 - ![open] groupBy, aggregation, and window functions
 - ![open] better error handling

examples/databricks.rs

@@ -0,0 +1,46 @@
+// This example demonstrates connecting to a Databricks Cluster via a
+// tls connection.
+//
+// This demo requires access to a Databricks Workspace, a personal access token,
+// and a cluster id. The cluster should be running a 13.3LTS runtime or greater. Populate
+// the remote URL string between the `<>` with the appropriate details.
+//
+// To view the connected Spark Session, go to the cluster Spark UI and select the 'Connect' tab.
+
+use spark_connect_rs::{SparkSession, SparkSessionBuilder};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let spark: SparkSession = SparkSessionBuilder::remote("sc://<workspace id>:443/;token=<personal access token>;x-databricks-cluster-id=<cluster-id>")
+        .build()
+        .await?;
+
+    spark
+        .range(None, 10, 1, Some(1))
+        .selectExpr(vec!["id * 4"])
+        .show(Some(10), None, None)
+        .await
+        .unwrap();
+
+    // +-------------+
+    // | show_string |
+    // +-------------+
+    // | +--------+  |
+    // | |(id * 4)|  |
+    // | +--------+  |
+    // | |0       |  |
+    // | |4       |  |
+    // | |8       |  |
+    // | |12      |  |
+    // | |16      |  |
+    // | |20      |  |
+    // | |24      |  |
+    // | |28      |  |
+    // | |32      |  |
+    // | |36      |  |
+    // | +--------+  |
+    // |             |
+    // +-------------+
+
+    Ok(())
+}

src/catalog.rs

@@ -145,7 +145,7 @@ mod tests {
     async fn setup() -> SparkSession {
         println!("SparkSession Setup");
 
-        let connection = "sc://127.0.0.1:15002/;user_id=rust_test".to_string();
+        let connection = "sc://127.0.0.1:15002/;user_id=rust_test";
 
         SparkSessionBuilder::remote(connection)
             .build()