From 0a16a0f0c79f4d3338e8dc702c1f67a351c0b597 Mon Sep 17 00:00:00 2001 From: Phil Nguyen Date: Tue, 17 Sep 2024 11:26:07 +0700 Subject: [PATCH] Add maxKeys to S3 list API --- src/main/api/studio-api.yaml | 6 +++++ .../api/v2/service/aws/s3/AwsS3Service.java | 5 ++-- .../controller/rest/v2/RequestConstants.java | 3 ++- .../rest/v2/aws/AwsS3Controller.java | 9 ++++--- .../v2/service/aws/s3/AwsS3ServiceImpl.java | 26 +++++++++++++++---- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/main/api/studio-api.yaml b/src/main/api/studio-api.yaml index a94011b058..94d750f8ad 100644 --- a/src/main/api/studio-api.yaml +++ b/src/main/api/studio-api.yaml @@ -1561,6 +1561,12 @@ paths: required: false schema: type: string + - name: maxKeys + in: query + description: The maximum number of keys to retrieve (default value is 100) + required: false + schema: + type: integer responses: '200': description: OK diff --git a/src/main/java/org/craftercms/studio/api/v2/service/aws/s3/AwsS3Service.java b/src/main/java/org/craftercms/studio/api/v2/service/aws/s3/AwsS3Service.java index a494489f0d..e27261cf7f 100644 --- a/src/main/java/org/craftercms/studio/api/v2/service/aws/s3/AwsS3Service.java +++ b/src/main/java/org/craftercms/studio/api/v2/service/aws/s3/AwsS3Service.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2007-2022 Crafter Software Corporation. All Rights Reserved. + * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 as published by @@ -52,12 +52,13 @@ S3Item uploadItem(String siteId, String profileId, String path, String filename, * @param profileId the profile id * @param path the path to list * @param type the type of items to list + * @param maxKeys the maximum number of keys * @return the list of items * @throws AwsException if there is any error connection to S3 * @throws SiteNotFoundException if the site is not found * @throws ConfigurationProfileNotFoundException if the profile is not found */ - List listItems(String siteId, String profileId, String path, String type) + List listItems(String siteId, String profileId, String path, String type, int maxKeys) throws AwsException, SiteNotFoundException, ConfigurationProfileNotFoundException; } diff --git a/src/main/java/org/craftercms/studio/controller/rest/v2/RequestConstants.java b/src/main/java/org/craftercms/studio/controller/rest/v2/RequestConstants.java index e1282ea8e9..df879e409f 100644 --- a/src/main/java/org/craftercms/studio/controller/rest/v2/RequestConstants.java +++ b/src/main/java/org/craftercms/studio/controller/rest/v2/RequestConstants.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2007-2022 Crafter Software Corporation. All Rights Reserved. + * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 as published by @@ -55,6 +55,7 @@ public final class RequestConstants { public static final String REQUEST_PARAM_PUBLISHING_TARGET = "publishingTarget"; public static final String REQUEST_PARAM_APPROVER = "approver"; public static final String REQUEST_PARAM_ITEM_TYPE = "itemType"; + public static final String REQUEST_PARAM_S3_MAX_KEYS= "maxKeys"; public static final String GROUP_SORT_COLUMNS = "id record_last_updated group_name externally_managed"; diff --git a/src/main/java/org/craftercms/studio/controller/rest/v2/aws/AwsS3Controller.java b/src/main/java/org/craftercms/studio/controller/rest/v2/aws/AwsS3Controller.java index b750806687..0a05c3bb48 100644 --- a/src/main/java/org/craftercms/studio/controller/rest/v2/aws/AwsS3Controller.java +++ b/src/main/java/org/craftercms/studio/controller/rest/v2/aws/AwsS3Controller.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2007-2022 Crafter Software Corporation. All Rights Reserved. + * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 as published by @@ -16,6 +16,7 @@ package org.craftercms.studio.controller.rest.v2.aws; +import jakarta.validation.constraints.PositiveOrZero; import org.apache.commons.fileupload2.core.FileUploadException; import org.apache.commons.fileupload2.core.FileItemInput; import org.apache.commons.fileupload2.core.FileItemInputIterator; @@ -74,6 +75,7 @@ public class AwsS3Controller { * @param profileId the profile id * @param path the path to list * @param type the type of file to list + * @param maxKeys the maximum number of keys * @return the list of items * @throws AwsException if there is any error connecting to S3 * @throws SiteNotFoundException if the site is not found @@ -84,11 +86,12 @@ public ResultList listItems( @ValidSiteId @RequestParam(REQUEST_PARAM_SITEID) String siteId, @ValidateNoTagsParam @RequestParam(REQUEST_PARAM_PROFILE_ID) String profileId, @ValidExistingContentPath @RequestParam(value = REQUEST_PARAM_PATH, required = false, defaultValue = StringUtils.EMPTY) String path, - @ValidateNoTagsParam @RequestParam(value = REQUEST_PARAM_TYPE, required = false, defaultValue = StringUtils.EMPTY) String type) + @ValidateNoTagsParam @RequestParam(value = REQUEST_PARAM_TYPE, required = false, defaultValue = StringUtils.EMPTY) String type, + @PositiveOrZero @RequestParam(value = REQUEST_PARAM_S3_MAX_KEYS, required = false, defaultValue = "100") int maxKeys) throws AwsException, SiteNotFoundException, ConfigurationProfileNotFoundException { ResultList result = new ResultList<>(); - result.setEntities(RESULT_KEY_ITEMS, s3Service.listItems(siteId, profileId, path, type)); + result.setEntities(RESULT_KEY_ITEMS, s3Service.listItems(siteId, profileId, path, type, maxKeys)); result.setResponse(ApiResponse.OK); return result; diff --git a/src/main/java/org/craftercms/studio/impl/v2/service/aws/s3/AwsS3ServiceImpl.java b/src/main/java/org/craftercms/studio/impl/v2/service/aws/s3/AwsS3ServiceImpl.java index fcb8102ff3..ea558731a7 100644 --- a/src/main/java/org/craftercms/studio/impl/v2/service/aws/s3/AwsS3ServiceImpl.java +++ b/src/main/java/org/craftercms/studio/impl/v2/service/aws/s3/AwsS3ServiceImpl.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2007-2023 Crafter Software Corporation. All Rights Reserved. + * Copyright (C) 2007-2024 Crafter Software Corporation. All Rights Reserved. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 as published by @@ -129,7 +129,8 @@ public S3Item uploadItem(@SiteId String siteId, public List listItems(@SiteId String siteId, @ValidateStringParam String profileId, @ValidateStringParam String path, - @ValidateStringParam String type) throws AwsException, + @ValidateStringParam String type, + int maxKeys) throws AwsException, SiteNotFoundException, ConfigurationProfileNotFoundException { S3Profile profile = getProfile(siteId, profileId); S3Client client = getS3Client(profile); @@ -147,6 +148,8 @@ public List listItems(@SiteId String siteId, .build(); ListObjectsV2Iterable response = client.listObjectsV2Paginator(request); + int commonPrefixesCount = 0; + // fetch all prefixes and fetch content key up to maxKeys for (ListObjectsV2Response page : response) { page.commonPrefixes().stream() .map(p -> { @@ -154,15 +157,28 @@ public List listItems(@SiteId String siteId, return new S3Item(StringUtils.removeEnd(relativeKey, delimiter), relativeKey, true, profile.getBucketName(), profile.getPrefix()); }) .forEach(items::add); + commonPrefixesCount += page.commonPrefixes().size(); - page.contents().stream() + // Do not fetch content key if it exceeded the maxKeys but continue to fetch prefixes + if (items.size() >= maxKeys + commonPrefixesCount) { + continue; + } + + List contents = page.contents().stream() .filter(o -> !StringUtils.equals(o.key(), fullPrefix) && MimeType.valueOf(StudioUtils.getMimeType(o.key())).isCompatibleWith(filerType)) .map(o -> { String relativeKey = StringUtils.removeStart(o.key(), profile.getPrefix()); return new S3Item(relativeKey, createUrl(profileId, relativeKey), false, profile.getBucketName(), profile.getPrefix()); - }) - .forEach(items::add); + }).toList(); + + for (S3Item content: contents) { + // Do not add more content key if the total exceeded the maxKeys + if (items.size() >= maxKeys + commonPrefixesCount) { + break; + } + items.add(content); + } } return items;