Merge pull request #49 from 3KeyCompany/release/1.5.0

README.md

@@ -84,15 +84,17 @@ For more information, please refer to the [CZERTAINLY documentation](https://doc
 
 EJBCA NG `Connector` is provided as a Docker container. Use the `docker pull harbor.3key.company/czertainly/czertainly-ejbca-ng-connector:tagname` to pull the required image from the repository. It can be configured using the following environment variables:
 
-| Variable               | Description                                         | Required                                           | Default value |
-|------------------------|-----------------------------------------------------|----------------------------------------------------|---------------|
-| `JDBC_URL`             | JDBC URL for database access                        | ![](https://img.shields.io/badge/-YES-success.svg) | `N/A`         |
-| `JDBC_USERNAME`        | Username to access the database                     | ![](https://img.shields.io/badge/-YES-success.svg) | `N/A`         |
-| `JDBC_PASSWORD`        | Password to access the database                     | ![](https://img.shields.io/badge/-YES-success.svg) | `N/A`         |
-| `DB_SCHEMA`            | Database schema to use                              | ![](https://img.shields.io/badge/-NO-red.svg)      | `ejbca`       |
-| `PORT`                 | Port where the service is exposed                   | ![](https://img.shields.io/badge/-NO-red.svg)      | `8080`        |
-| `TRUSTED_CERTIFICATES` | List of PEM encoded additional trusted certificates | ![](https://img.shields.io/badge/-NO-red.svg)      | `N/A`         |
-| `REMOTE_DEBUG`         | Enables JVM remote debug on port 5005               | ![](https://img.shields.io/badge/-NO-red.svg)      | `false`       |
+| Variable                 | Description                                            | Required                                           | Default value |
+|--------------------------|--------------------------------------------------------|----------------------------------------------------|---------------|
+| `JDBC_URL`               | JDBC URL for database access                           | ![](https://img.shields.io/badge/-YES-success.svg) | `N/A`         |
+| `JDBC_USERNAME`          | Username to access the database                        | ![](https://img.shields.io/badge/-YES-success.svg) | `N/A`         |
+| `JDBC_PASSWORD`          | Password to access the database                        | ![](https://img.shields.io/badge/-YES-success.svg) | `N/A`         |
+| `DB_SCHEMA`              | Database schema to use                                 | ![](https://img.shields.io/badge/-NO-red.svg)      | `ejbca`       |
+| `PORT`                   | Port where the service is exposed                      | ![](https://img.shields.io/badge/-NO-red.svg)      | `8080`        |
+| `TRUSTED_CERTIFICATES`   | List of PEM encoded additional trusted certificates    | ![](https://img.shields.io/badge/-NO-red.svg)      | `N/A`         |
+| `REMOTE_DEBUG`           | Enables JVM remote debug on port 5005                  | ![](https://img.shields.io/badge/-NO-red.svg)      | `false`       |
+| `MAX_PAYLOAD_SIZE`       | Maximum payload size in bytes                          | ![](https://img.shields.io/badge/-NO-red.svg)      | `2000000`     |
+| `EJBCA_SEARCH_PAGE_SIZE` | Maximum number of certificates to fetch in one request | ![](https://img.shields.io/badge/-NO-red.svg)      | `100`         |
 
 ### Proxy settings
 

pom.xml

@@ -13,7 +13,7 @@
 
     <groupId>com.czertainly</groupId>
     <artifactId>ejbca-ng-connector</artifactId>
-    <version>1.4.0</version>
+    <version>1.5.0</version>
     <name>CZERTAINLY-EJBCA-NG-Connector</name>
 
     <properties>
@@ -40,7 +40,7 @@
         <dependency>
             <groupId>com.czertainly</groupId>
             <artifactId>interfaces</artifactId>
-            <version>2.7.0</version>
+            <version>2.8.1</version>
         </dependency>
 
 <!--        <dependency>-->

src/main/java/com/czertainly/ca/connector/ejbca/api/DiscoveryControllerImpl.java

@@ -34,15 +34,15 @@ public void setDiscoveryService(DiscoveryService discoveryService) {
 
     @Override
     public DiscoveryProviderDto discoverCertificate(DiscoveryRequestDto request) throws IOException, NotFoundException {
-        logger.info("Initiating certificate discovery for the given inputs");
+        logger.debug("Initiating certificate discovery: {}", request);
         DiscoveryHistory history;
         history = discoveryHistoryService.addHistory(request);
         discoveryService.discoverCertificate(request, history);
         DiscoveryDataRequestDto dto = new DiscoveryDataRequestDto();
         dto.setName(request.getName());
         // initial values when discovery is finished before the Async method discoverCertificate
-        dto.setStartIndex(0);
-        dto.setEndIndex(1000);
+        dto.setPageNumber(0);
+        dto.setItemsPerPage(10);
         return discoveryService.getProviderDtoData(dto, history);
     }
 

src/main/java/com/czertainly/ca/connector/ejbca/service/impl/AttributeServiceImpl.java

@@ -29,7 +29,7 @@ public class AttributeServiceImpl implements AttributeService {
 
 	@Override
 	public List<BaseAttribute> getAttributes(String kind) {
-		logger.info("Getting the attributes for {}", kind);
+		logger.debug("Getting the attributes for {}", kind);
 		List<BaseAttribute> attrs = new ArrayList<>();
 
         DataAttribute url = new DataAttribute();

src/main/java/com/czertainly/ca/connector/ejbca/service/impl/AuthorityInstanceServiceImpl.java

@@ -58,6 +58,12 @@
 public class AuthorityInstanceServiceImpl implements AuthorityInstanceService {
     private static final Logger logger = LoggerFactory.getLogger(AuthorityInstanceServiceImpl.class);
 
+    /**
+     * This is the maximum size in bytes of the payload
+     */
+    @Value("${spring.codec.max-in-memory-size:2000000}")
+    private int MAX_PAYLOAD_SIZE;
+
     private static final Map<Long, EjbcaWS> connectionsCache = new ConcurrentHashMap<>();
     private static final Map<Long, WebClient> connectionsRestApiCache = new ConcurrentHashMap<>();
 
@@ -318,12 +324,8 @@ public String getRestApiUrl(String authorityInstanceUuid) throws NotFoundExcepti
     private WebClient createRestApiConnection(AuthorityInstance instance) {
         List<BaseAttribute> attributes = AttributeDefinitionUtils.deserialize(instance.getCredentialData(), BaseAttribute.class);
 
-        /**
-         * 1 certificate in response ~ 2000 bytes * 1000 = 2000000
-         */
-        final int size = 2000 * 1000;
         final ExchangeStrategies strategies = ExchangeStrategies.builder()
-                .codecs(codecs -> codecs.defaultCodecs().maxInMemorySize(size))
+                .codecs(codecs -> codecs.defaultCodecs().maxInMemorySize(MAX_PAYLOAD_SIZE))
                 .build();
 
         SslContext sslContext = EjbcaRestApiClient.createSslContext(attributes, trustedCertificatesConfig.getDefaultTrustManagers());

src/main/java/com/czertainly/ca/connector/ejbca/service/impl/DiscoveryAttributeServiceImpl.java

@@ -64,7 +64,7 @@ public List<BaseAttribute> getAttributes(String kind) {
         if (!kind.equals(DiscoveryKind.EJBCA.name()) && !kind.equals(DiscoveryKind.EJBCA_SCHEDULE.name())) {
             throw new ValidationException("Unsupported kind: " + kind, new ValidationError("Unsupported kind: " + kind));
         }
-        logger.info("Listing discovery attributes for {}", kind);
+        logger.debug("Listing discovery attributes for {}", kind);
 
         List<BaseAttribute> attributes = new ArrayList<>();
         attributes.add(infoDiscoveryDescription());
@@ -384,8 +384,6 @@ private InfoAttribute infoDiscoveryDescription() {
                 .append(new Text("Select EJBCA instance where Discovery process should search for Certificates and then you can optionally select:")).append("\n")
                 .append(new UnorderedList<>(items));
 
-        System.out.println(sb);
-
         attribute.setContent(List.of(new TextAttributeContent(sb.toString())));
 
         return attribute;

src/main/java/com/czertainly/ca/connector/ejbca/service/impl/DiscoveryServiceImpl.java

@@ -27,10 +27,12 @@
 import com.czertainly.ca.connector.ejbca.service.DiscoveryHistoryService;
 import com.czertainly.ca.connector.ejbca.service.DiscoveryService;
 import com.czertainly.ca.connector.ejbca.service.EjbcaService;
+import com.czertainly.ca.connector.ejbca.util.EjbcaVersion;
 import com.czertainly.core.util.AttributeDefinitionUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
 import org.springframework.scheduling.annotation.Async;
@@ -56,7 +58,8 @@ public class DiscoveryServiceImpl implements DiscoveryService {
     /**
      * This constant represents the number of certificate per page in searching
      */
-    private static final int EJBCA_SEARCH_PAGE_SIZE = 1000;
+    @Value("${ejbca.search.pageSize:100}")
+    private int EJBCA_SEARCH_PAGE_SIZE;
     private EjbcaService ejbcaService;
     private CertificateRepository certificateRepository;
     private DiscoveryHistoryService discoveryHistoryService;
@@ -102,7 +105,7 @@ public DiscoveryProviderDto getProviderDtoData(DiscoveryDataRequestDto request,
             dto.setCertificateData(new ArrayList<>());
             dto.setTotalCertificatesDiscovered(0);
         } else {
-            Pageable page = PageRequest.of(request.getStartIndex(), request.getEndIndex());
+            Pageable page = PageRequest.of(request.getPageNumber() <= 0 ? 0 : request.getPageNumber() - 1, request.getItemsPerPage());
             dto.setCertificateData(certificateRepository.findAllByDiscoveryId(history.getId(), page).stream().map(Certificate::mapToDto).collect(Collectors.toList()));
         }
         return dto;
@@ -160,17 +163,39 @@ private void discoverCertificatesInternal(DiscoveryRequestDto request, Discovery
 
         SearchCertificatesRestRequestV2 searchRequest = prepareSearchRequest(cas, eeProfiles, statuses, issuedAfter);
         SearchCertificatesRestResponseV2 searchResponse;
-        do {
-            searchResponse = ejbcaService.searchCertificates(instance.getUuid(), restApiUrl, searchRequest);
-            // break the loop if there are no certificates returned from EJBCA
-            if (searchResponse.getCertificates().isEmpty()) {
-                break;
-            }
-            // set the next page
-            searchRequest.getPagination().setCurrentPage(searchResponse.getPaginationSummary().getCurrentPage() + 1);
-            parseAndCreateCertificateEntry(searchResponse, history);
-            certificatesFound = certificatesFound + searchResponse.getCertificates().size();
-        } while (searchResponse.getPaginationSummary().getTotalCerts() == null);
+
+        // behaviour of the EJBCA REST API for searching certificates differs between versions
+        // we need to check the version and decide on the implementation
+        // TODO: this can be improved once there are more versions of implementation
+        EjbcaVersion ejbcaVersion = ejbcaService.getEjbcaVersion(instance.getUuid());
+        logger.debug("Searching for certificates in EJBCA version {}, with page size {}", ejbcaVersion.getVersion(), EJBCA_SEARCH_PAGE_SIZE);
+
+        // when the version is at least 7.11
+        if (ejbcaVersion.getTechVersion() >= 7 && ejbcaVersion.getMajorVersion() >= 11) {
+            do {
+                searchResponse = ejbcaService.searchCertificates(instance.getUuid(), restApiUrl, searchRequest);
+                // break the loop if there are no certificates returned from EJBCA
+                if (searchResponse.getCertificates().isEmpty()) {
+                    break;
+                }
+                // set the next page
+                searchRequest.getPagination().setCurrentPage(searchResponse.getPaginationSummary().getCurrentPage() + 1);
+                parseAndCreateCertificateEntry(searchResponse, history);
+                certificatesFound = certificatesFound + searchResponse.getCertificates().size();
+            } while (!searchResponse.getCertificates().isEmpty());
+        } else { // when the version is lower than 7.11, but higher than 7.8
+            do {
+                searchResponse = ejbcaService.searchCertificates(instance.getUuid(), restApiUrl, searchRequest);
+                // break the loop if there are no certificates returned from EJBCA
+                if (searchResponse.getCertificates().isEmpty()) {
+                    break;
+                }
+                // set the next page
+                searchRequest.getPagination().setCurrentPage(searchResponse.getPaginationSummary().getCurrentPage() + 1);
+                parseAndCreateCertificateEntry(searchResponse, history);
+                certificatesFound = certificatesFound + searchResponse.getCertificates().size();
+            } while (searchResponse.getPaginationSummary().getTotalCerts() == null);
+        }
 
         history.setStatus(DiscoveryStatus.COMPLETED);
         history.setMeta(AttributeDefinitionUtils.serialize(getDiscoveryMeta(certificatesFound)));

src/main/java/com/czertainly/ca/connector/ejbca/util/EjbcaVersion.java

@@ -7,8 +7,8 @@ public class EjbcaVersion {
     private int minorVersion;
     private String version;
 
-    public EjbcaVersion(String ejbcaersion) {
-        readVersionNumbers(ejbcaersion);
+    public EjbcaVersion(String ejbcaVersion) {
+        readVersionNumbers(ejbcaVersion);
     }
 
     private void readVersionNumbers(String ejbcaVersion) {

src/main/resources/application.yml

@@ -16,10 +16,17 @@ spring:
   jpa:
     properties:
       hibernate.default_schema: ${DB_SCHEMA:ejbca}
+  codec:
+    # The maximum number of bytes that can be buffered whenever a codec is used.
+    max-in-memory-size: ${MAX_PAYLOAD_SIZE:2000000}
+
 ejbca:
   timeout:
     connect: 500
     request: 1500
+  search:
+    # The maximum number of certificates to return in one page
+    pageSize: ${EJBCA_SEARCH_PAGE_SIZE:100}
 
 management:
   endpoints: