String hash table references instead of nested tables.

entur · Oct 17, 2024 · a0d7909 · a0d7909
1 parent 365eb76
commit a0d7909
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 31 deletions.
diff --git a/src/main/java/no/entur/antu/config/NetexDataConfig.java b/src/main/java/no/entur/antu/config/NetexDataConfig.java
@@ -11,6 +11,7 @@
 import no.entur.antu.netexdata.DefaultNetexDataRepository;
 import no.entur.antu.netexdata.NetexDataResource;
 import org.entur.netex.validation.validator.jaxb.NetexDataRepository;
+import org.redisson.api.RedissonClient;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -29,6 +30,7 @@ NetexDataResource netexDataResource() {
   @Profile("!test")
   NetexDataRepository netexDataRepository(
     NetexDataResource netexDataResource,
+    RedissonClient redissonClient,
     @Qualifier(
       SCHEDULED_STOP_POINT_AND_QUAY_ID_CACHE
     ) Map<String, Map<String, String>> scheduledStopPointAndQuayIdCache,
@@ -45,6 +47,7 @@ NetexDataRepository netexDataRepository(
   ) {
     return new DefaultNetexDataRepository(
       netexDataResource,
+      redissonClient,
       scheduledStopPointAndQuayIdCache,
       serviceLinksAndFromToScheduledStopPointIdCache,
       lineInfoCache,

diff --git a/src/main/java/no/entur/antu/config/TimetableDataValidatorConfig.java b/src/main/java/no/entur/antu/config/TimetableDataValidatorConfig.java
@@ -324,14 +324,14 @@ public NetexValidatorsRunner timetableDataValidatorsRunner(
     );
 
     List<DatasetValidator> netexTimetableDatasetValidators = List.of(
-      duplicateLineNameValidator
-      //      stopPointsInVehicleJourneyValidator
+      duplicateLineNameValidator,
+      stopPointsInVehicleJourneyValidator
     );
 
     List<NetexDataCollector> commonDataCollectors = List.of(
       lineInfoCollector,
-      //      serviceJourneyStopsCollector,
-      serviceJourneyInterchangeInfoCollector
+      serviceJourneyInterchangeInfoCollector,
+      serviceJourneyStopsCollector
     );
 
     return new NetexValidatorsRunner(

diff --git a/src/main/java/no/entur/antu/netexdata/DefaultNetexDataRepository.java b/src/main/java/no/entur/antu/netexdata/DefaultNetexDataRepository.java
@@ -2,11 +2,13 @@
 
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
+import java.util.Map.Entry;
 import java.util.stream.Collectors;
 import no.entur.antu.exception.AntuException;
 import org.entur.netex.validation.validator.jaxb.*;
 import org.entur.netex.validation.validator.model.*;
+import org.redisson.RedissonLocalCachedMap;
+import org.redisson.api.RedissonClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -21,6 +23,7 @@ public class DefaultNetexDataRepository implements NetexDataRepository {
   );
 
   private final NetexDataResource netexDataResource;
+  private final RedissonClient redissonClient;
   private final Map<String, Map<String, String>> scheduledStopPointAndQuayIdCache;
   private final Map<String, Map<String, String>> serviceLinksAndFromToScheduledStopPointIdCache;
   private final Map<String, List<String>> lineInfoCache;
@@ -29,13 +32,15 @@ public class DefaultNetexDataRepository implements NetexDataRepository {
 
   public DefaultNetexDataRepository(
     NetexDataResource netexDataResource,
+    RedissonClient redissonClient,
     Map<String, Map<String, String>> scheduledStopPointAndQuayIdCache,
     Map<String, Map<String, String>> serviceLinksAndFromToScheduledStopPointIdCache,
     Map<String, List<String>> lineInfoCache,
     Map<String, Map<String, List<String>>> serviceJourneyStopsCache,
     Map<String, List<String>> serviceJourneyInterchangeInfoCache
   ) {
     this.netexDataResource = netexDataResource;
+    this.redissonClient = redissonClient;
     this.scheduledStopPointAndQuayIdCache = scheduledStopPointAndQuayIdCache;
     this.serviceLinksAndFromToScheduledStopPointIdCache =
       serviceLinksAndFromToScheduledStopPointIdCache;
@@ -105,24 +110,20 @@ public List<ServiceJourneyStop> serviceJourneyStops(
     String validationReportId,
     ServiceJourneyId serviceJourneyId
   ) {
-    Map<String, List<String>> serviceJourneyStopsForReport =
-      serviceJourneyStopsCache.get(validationReportId);
-    if (serviceJourneyStopsForReport == null) {
-      throw new AntuException(
-        "ServiceJourneyStops cache not found for validation report with id: " +
-        validationReportId
-      );
+    if (
+      serviceJourneyStopsCache instanceof RedissonLocalCachedMap<String, Map<String, List<String>>> localCachedMap
+    ) {
+      return localCachedMap
+        .keySet(validationReportId + "*")
+        .stream()
+        .map(localCachedMap::get)
+        .flatMap(m -> m.entrySet().stream())
+        .filter(e -> e.getKey().equals(serviceJourneyId.id()))
+        .flatMap(e -> e.getValue().stream())
+        .map(ServiceJourneyStop::fromString)
+        .toList();
     }
-    return Optional
-      .ofNullable(serviceJourneyStopsForReport.get(serviceJourneyId.id()))
-      .map(serviceJourneyStops ->
-        serviceJourneyStops
-          .stream()
-          .map(ServiceJourneyStop::fromString)
-          .filter(ServiceJourneyStop::isValid)
-          .toList()
-      )
-      .orElse(List.of());
+    return List.of();
   }
 
   @Override
@@ -176,7 +177,7 @@ public void fillNetexDataCache(
         .getFromToScheduledStopPointIdPerServiceLinkId()
         .entrySet()
         .stream()
-        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
 
     serviceLinksAndFromToScheduledStopPointIdCache.merge(
       validationReportId,
@@ -199,6 +200,10 @@ public void cleanUp(String validationReportId) {
     scheduledStopPointAndQuayIdCache.remove(validationReportId);
     serviceLinksAndFromToScheduledStopPointIdCache.remove(validationReportId);
     lineInfoCache.remove(validationReportId);
+    redissonClient.getKeys().deleteByPattern(validationReportId + '*');
+    serviceJourneyStopsCache
+      .keySet()
+      .removeIf(k -> k.startsWith(validationReportId));
     serviceJourneyStopsCache.remove(validationReportId);
     serviceJourneyInterchangeInfoCache.remove(validationReportId);
   }

diff --git a/src/main/java/no/entur/antu/netexdata/collectors/ServiceJourneyStopsCollector.java b/src/main/java/no/entur/antu/netexdata/collectors/ServiceJourneyStopsCollector.java
@@ -9,6 +9,7 @@
 import org.entur.netex.validation.validator.model.ScheduledStopPointId;
 import org.entur.netex.validation.validator.model.ServiceJourneyStop;
 import org.redisson.api.RLock;
+import org.redisson.api.RMap;
 import org.redisson.api.RedissonClient;
 import org.springframework.stereotype.Component;
 
@@ -44,7 +45,6 @@ protected void collectDataFromLineFile(
 
     Map<String, List<String>> serviceJourneyStops = antuNetexData
       .validServiceJourneys()
-      // TODO: unique service journeys ids
       .map(serviceJourney -> {
         Map<String, ScheduledStopPointId> scheduledStopPointIdMap =
           AntuNetexData.scheduledStopPointIdByStopPointId(
@@ -71,6 +71,7 @@ protected void collectDataFromLineFile(
 
     addServiceJourneyStops(
       validationContext.getValidationReportId(),
+      validationContext.getFileName(),
       serviceJourneyStops
     );
   }
@@ -84,19 +85,20 @@ protected void collectDataFromCommonFile(
 
   private void addServiceJourneyStops(
     String validationReportId,
+    String filename,
     Map<String, List<String>> serviceJourneyStops
   ) {
     RLock lock = redissonClient.getLock(validationReportId);
     try {
       lock.lock();
 
-      serviceJourneyStopsCache.merge(
-        validationReportId,
-        serviceJourneyStops,
-        (existingMap, newMap) -> {
-          existingMap.putAll(newMap);
-          return existingMap;
-        }
+      RMap<String, List<String>> serviceJourneyStopsMap = redissonClient.getMap(
+        validationReportId + "_" + filename
+      );
+      serviceJourneyStopsMap.putAll(serviceJourneyStops);
+      serviceJourneyStopsCache.put(
+        validationReportId + "_" + filename,
+        serviceJourneyStopsMap
       );
     } finally {
       if (lock.isHeldByCurrentThread()) {