Revert "Revert "S3 CRT Client (#598)" (#609)"

This reverts commit 6728de8.
slackhq · Jul 14, 2023 · e3ad2c0 · e3ad2c0
1 parent 6728de8
commit e3ad2c0
Show file tree

Hide file tree

Showing 21 changed files with 793 additions and 185 deletions.
diff --git a/config/config.yaml b/config/config.yaml
@@ -32,6 +32,7 @@ s3Config:
   s3Region: ${S3_REGION:-us-east-1}
   s3EndPoint: ${S3_ENDPOINT:-http://localhost:9090}
   s3Bucket: ${S3_BUCKET:-test-s3-bucket}
+  s3TargetThroughputGbps: ${S3_TARGET_THROUGHPUT_GBPS:-25}
 
 tracingConfig:
   zipkinEndpoint: ${ZIPKIN_TRACING_ENDPOINT:-http://localhost:9411/api/v2/spans}

diff --git a/kaldb/pom.xml b/kaldb/pom.xml
@@ -313,6 +313,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk.crt</groupId>
+            <artifactId>aws-crt</artifactId>
+            <version>0.22.1</version>
+        </dependency>
 
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>

diff --git a/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3BlobFs.java b/kaldb/src/main/java/com/slack/kaldb/blobfs/s3/S3BlobFs.java
@@ -47,6 +47,10 @@
 import software.amazon.awssdk.services.s3.model.S3Exception;
 import software.amazon.awssdk.services.s3.model.S3Object;
 
+/**
+ * @see S3CrtBlobFs
+ */
+@Deprecated
 public class S3BlobFs extends BlobFs {
   public static final String S3_SCHEME = "s3://";
   private static final Logger LOG = LoggerFactory.getLogger(S3BlobFs.class);
@@ -226,7 +230,7 @@ private boolean copyFile(URI srcUri, URI dstUri) throws IOException {
 
   @Override
   public boolean mkdir(URI uri) throws IOException {
-    LOG.info("mkdir {}", uri);
+    LOG.debug("mkdir {}", uri);
     try {
       Preconditions.checkNotNull(uri, "uri is null");
       String path = normalizeToDirectoryPrefix(uri);
@@ -249,7 +253,7 @@ public boolean mkdir(URI uri) throws IOException {
 
   @Override
   public boolean delete(URI segmentUri, boolean forceDelete) throws IOException {
-    LOG.info("Deleting uri {} force {}", segmentUri, forceDelete);
+    LOG.debug("Deleting uri {} force {}", segmentUri, forceDelete);
     try {
       if (isDirectory(segmentUri)) {
         if (!forceDelete) {
@@ -312,7 +316,7 @@ public boolean doMove(URI srcUri, URI dstUri) throws IOException {
 
   @Override
   public boolean copy(URI srcUri, URI dstUri) throws IOException {
-    LOG.info("Copying uri {} to uri {}", srcUri, dstUri);
+    LOG.debug("Copying uri {} to uri {}", srcUri, dstUri);
     Preconditions.checkState(exists(srcUri), "Source URI '%s' does not exist", srcUri);
     if (srcUri.equals(dstUri)) {
       return true;
@@ -411,7 +415,7 @@ public String[] listFiles(URI fileUri, boolean recursive) throws IOException {
         continuationToken = listObjectsV2Response.nextContinuationToken();
       }
       String[] listedFiles = builder.build().toArray(new String[0]);
-      LOG.info(
+      LOG.debug(
           "Listed {} files from URI: {}, is recursive: {}", listedFiles.length, fileUri, recursive);
       return listedFiles;
     } catch (Throwable t) {