Add maxKeys to S3 list API

src/main/api/studio-api.yaml

@@ -1561,6 +1561,12 @@ paths:
           required: false
           schema:
             type: string
+        - name: maxKeys
+          in: query
+          description: The maximum number of keys to retrieve (default value is 100)
+          required: false
+          schema:
+            type: integer
       responses:
         '200':
           description: OK

src/main/java/org/craftercms/studio/api/v2/service/aws/s3/AwsS3Service.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2022 Crafter Software Corporation. All Rights Reserved.
+ * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as published by
@@ -52,12 +52,13 @@ S3Item uploadItem(String siteId, String profileId, String path, String filename,
      * @param profileId the profile id
      * @param path the path to list
      * @param type the type of items to list
+     * @param maxKeys the maximum number of keys
      * @return the list of items
      * @throws AwsException if there is any error connection to S3
      * @throws SiteNotFoundException if the site is not found
      * @throws ConfigurationProfileNotFoundException if the profile is not found
      */
-    List<S3Item> listItems(String siteId, String profileId, String path, String type)
+    List<S3Item> listItems(String siteId, String profileId, String path, String type, int maxKeys)
             throws AwsException, SiteNotFoundException, ConfigurationProfileNotFoundException;
 
 }

src/main/java/org/craftercms/studio/controller/rest/v2/RequestConstants.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2022 Crafter Software Corporation. All Rights Reserved.
+ * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as published by
@@ -55,6 +55,7 @@ public final class RequestConstants {
     public static final String REQUEST_PARAM_PUBLISHING_TARGET = "publishingTarget";
     public static final String REQUEST_PARAM_APPROVER = "approver";
     public static final String REQUEST_PARAM_ITEM_TYPE = "itemType";
+    public static final String REQUEST_PARAM_S3_MAX_KEYS= "maxKeys";
 
     public static final String GROUP_SORT_COLUMNS = "id record_last_updated group_name externally_managed";
 

src/main/java/org/craftercms/studio/controller/rest/v2/aws/AwsS3Controller.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2022 Crafter Software Corporation. All Rights Reserved.
+ * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as published by
@@ -16,6 +16,7 @@
 
 package org.craftercms.studio.controller.rest.v2.aws;
 
+import jakarta.validation.constraints.PositiveOrZero;
 import org.apache.commons.fileupload2.core.FileUploadException;
 import org.apache.commons.fileupload2.core.FileItemInput;
 import org.apache.commons.fileupload2.core.FileItemInputIterator;
@@ -74,6 +75,7 @@ public class AwsS3Controller {
      * @param profileId the profile id
      * @param path      the path to list
      * @param type      the type of file to list
+     * @param maxKeys   the maximum number of keys
      * @return the list of items
      * @throws AwsException                          if there is any error connecting to S3
      * @throws SiteNotFoundException                 if the site is not found
@@ -84,11 +86,12 @@ public ResultList<S3Item> listItems(
             @ValidSiteId @RequestParam(REQUEST_PARAM_SITEID) String siteId,
             @ValidateNoTagsParam @RequestParam(REQUEST_PARAM_PROFILE_ID) String profileId,
             @ValidExistingContentPath @RequestParam(value = REQUEST_PARAM_PATH, required = false, defaultValue = StringUtils.EMPTY) String path,
-            @ValidateNoTagsParam @RequestParam(value = REQUEST_PARAM_TYPE, required = false, defaultValue = StringUtils.EMPTY) String type)
+            @ValidateNoTagsParam @RequestParam(value = REQUEST_PARAM_TYPE, required = false, defaultValue = StringUtils.EMPTY) String type,
+            @PositiveOrZero @RequestParam(value = REQUEST_PARAM_S3_MAX_KEYS, required = false, defaultValue = "100") int maxKeys)
             throws AwsException, SiteNotFoundException, ConfigurationProfileNotFoundException {
 
         ResultList<S3Item> result = new ResultList<>();
-        result.setEntities(RESULT_KEY_ITEMS, s3Service.listItems(siteId, profileId, path, type));
+        result.setEntities(RESULT_KEY_ITEMS, s3Service.listItems(siteId, profileId, path, type, maxKeys));
         result.setResponse(ApiResponse.OK);
 
         return result;

src/main/java/org/craftercms/studio/impl/v2/service/aws/s3/AwsS3ServiceImpl.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007-2023 Crafter Software Corporation. All Rights Reserved.
+ * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved.
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 3 as published by
@@ -129,7 +129,8 @@ public S3Item uploadItem(@SiteId String siteId,
     public List<S3Item> listItems(@SiteId String siteId,
                                   @ValidateStringParam String profileId,
                                   @ValidateStringParam String path,
-                                  @ValidateStringParam String type) throws AwsException,
+                                  @ValidateStringParam String type,
+                                  int maxKeys) throws AwsException,
             SiteNotFoundException, ConfigurationProfileNotFoundException {
         S3Profile profile = getProfile(siteId, profileId);
         S3Client client = getS3Client(profile);
@@ -147,22 +148,37 @@ public List<S3Item> listItems(@SiteId String siteId,
                 .build();
 
         ListObjectsV2Iterable response = client.listObjectsV2Paginator(request);
+        int commonPrefixesCount = 0;
+        // fetch all prefixes and fetch content key up to maxKeys
         for (ListObjectsV2Response page : response) {
             page.commonPrefixes().stream()
                     .map(p -> {
                         String relativeKey = StringUtils.removeStart(p.prefix(), profile.getPrefix());
                         return new S3Item(StringUtils.removeEnd(relativeKey, delimiter), relativeKey, true, profile.getBucketName(), profile.getPrefix());
                     })
                     .forEach(items::add);
+            commonPrefixesCount += page.commonPrefixes().size();
 
-            page.contents().stream()
+            // Do not fetch content key if it exceeded the maxKeys but continue to fetch prefixes
+            if (items.size() >= maxKeys + commonPrefixesCount) {
+                continue;
+            }
+
+            List<S3Item> contents = page.contents().stream()
                     .filter(o -> !StringUtils.equals(o.key(), fullPrefix) &&
                             MimeType.valueOf(StudioUtils.getMimeType(o.key())).isCompatibleWith(filerType))
                     .map(o -> {
                         String relativeKey = StringUtils.removeStart(o.key(), profile.getPrefix());
                         return new S3Item(relativeKey, createUrl(profileId, relativeKey), false, profile.getBucketName(), profile.getPrefix());
-                    })
-                    .forEach(items::add);
+                    }).toList();
+
+            for (S3Item content: contents) {
+                // Do not add more content key if the total exceeded the maxKeys
+                if (items.size() >= maxKeys + commonPrefixesCount) {
+                    break;
+                }
+                items.add(content);
+            }
         }
 
         return items;