From a0d79092cdf4952610f1a877dc5b21676704cd4f Mon Sep 17 00:00:00 2001 From: Mansoor Sajjad Date: Thu, 17 Oct 2024 22:53:58 +0200 Subject: [PATCH] String hash table references instead of nested tables. --- .../no/entur/antu/config/NetexDataConfig.java | 3 ++ .../config/TimetableDataValidatorConfig.java | 8 ++-- .../netexdata/DefaultNetexDataRepository.java | 43 +++++++++++-------- .../ServiceJourneyStopsCollector.java | 18 ++++---- 4 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/main/java/no/entur/antu/config/NetexDataConfig.java b/src/main/java/no/entur/antu/config/NetexDataConfig.java index f6a2591b..1935bf75 100644 --- a/src/main/java/no/entur/antu/config/NetexDataConfig.java +++ b/src/main/java/no/entur/antu/config/NetexDataConfig.java @@ -11,6 +11,7 @@ import no.entur.antu.netexdata.DefaultNetexDataRepository; import no.entur.antu.netexdata.NetexDataResource; import org.entur.netex.validation.validator.jaxb.NetexDataRepository; +import org.redisson.api.RedissonClient; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -29,6 +30,7 @@ NetexDataResource netexDataResource() { @Profile("!test") NetexDataRepository netexDataRepository( NetexDataResource netexDataResource, + RedissonClient redissonClient, @Qualifier( SCHEDULED_STOP_POINT_AND_QUAY_ID_CACHE ) Map> scheduledStopPointAndQuayIdCache, @@ -45,6 +47,7 @@ NetexDataRepository netexDataRepository( ) { return new DefaultNetexDataRepository( netexDataResource, + redissonClient, scheduledStopPointAndQuayIdCache, serviceLinksAndFromToScheduledStopPointIdCache, lineInfoCache, diff --git a/src/main/java/no/entur/antu/config/TimetableDataValidatorConfig.java b/src/main/java/no/entur/antu/config/TimetableDataValidatorConfig.java index 77b47293..aca439e6 100644 --- a/src/main/java/no/entur/antu/config/TimetableDataValidatorConfig.java +++ b/src/main/java/no/entur/antu/config/TimetableDataValidatorConfig.java @@ -324,14 +324,14 @@ public NetexValidatorsRunner timetableDataValidatorsRunner( ); List netexTimetableDatasetValidators = List.of( - duplicateLineNameValidator - // stopPointsInVehicleJourneyValidator + duplicateLineNameValidator, + stopPointsInVehicleJourneyValidator ); List commonDataCollectors = List.of( lineInfoCollector, - // serviceJourneyStopsCollector, - serviceJourneyInterchangeInfoCollector + serviceJourneyInterchangeInfoCollector, + serviceJourneyStopsCollector ); return new NetexValidatorsRunner( diff --git a/src/main/java/no/entur/antu/netexdata/DefaultNetexDataRepository.java b/src/main/java/no/entur/antu/netexdata/DefaultNetexDataRepository.java index a5c52a68..e19acc28 100644 --- a/src/main/java/no/entur/antu/netexdata/DefaultNetexDataRepository.java +++ b/src/main/java/no/entur/antu/netexdata/DefaultNetexDataRepository.java @@ -2,11 +2,13 @@ import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.Map.Entry; import java.util.stream.Collectors; import no.entur.antu.exception.AntuException; import org.entur.netex.validation.validator.jaxb.*; import org.entur.netex.validation.validator.model.*; +import org.redisson.RedissonLocalCachedMap; +import org.redisson.api.RedissonClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,6 +23,7 @@ public class DefaultNetexDataRepository implements NetexDataRepository { ); private final NetexDataResource netexDataResource; + private final RedissonClient redissonClient; private final Map> scheduledStopPointAndQuayIdCache; private final Map> serviceLinksAndFromToScheduledStopPointIdCache; private final Map> lineInfoCache; @@ -29,6 +32,7 @@ public class DefaultNetexDataRepository implements NetexDataRepository { public DefaultNetexDataRepository( NetexDataResource netexDataResource, + RedissonClient redissonClient, Map> scheduledStopPointAndQuayIdCache, Map> serviceLinksAndFromToScheduledStopPointIdCache, Map> lineInfoCache, @@ -36,6 +40,7 @@ public DefaultNetexDataRepository( Map> serviceJourneyInterchangeInfoCache ) { this.netexDataResource = netexDataResource; + this.redissonClient = redissonClient; this.scheduledStopPointAndQuayIdCache = scheduledStopPointAndQuayIdCache; this.serviceLinksAndFromToScheduledStopPointIdCache = serviceLinksAndFromToScheduledStopPointIdCache; @@ -105,24 +110,20 @@ public List serviceJourneyStops( String validationReportId, ServiceJourneyId serviceJourneyId ) { - Map> serviceJourneyStopsForReport = - serviceJourneyStopsCache.get(validationReportId); - if (serviceJourneyStopsForReport == null) { - throw new AntuException( - "ServiceJourneyStops cache not found for validation report with id: " + - validationReportId - ); + if ( + serviceJourneyStopsCache instanceof RedissonLocalCachedMap>> localCachedMap + ) { + return localCachedMap + .keySet(validationReportId + "*") + .stream() + .map(localCachedMap::get) + .flatMap(m -> m.entrySet().stream()) + .filter(e -> e.getKey().equals(serviceJourneyId.id())) + .flatMap(e -> e.getValue().stream()) + .map(ServiceJourneyStop::fromString) + .toList(); } - return Optional - .ofNullable(serviceJourneyStopsForReport.get(serviceJourneyId.id())) - .map(serviceJourneyStops -> - serviceJourneyStops - .stream() - .map(ServiceJourneyStop::fromString) - .filter(ServiceJourneyStop::isValid) - .toList() - ) - .orElse(List.of()); + return List.of(); } @Override @@ -176,7 +177,7 @@ public void fillNetexDataCache( .getFromToScheduledStopPointIdPerServiceLinkId() .entrySet() .stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); serviceLinksAndFromToScheduledStopPointIdCache.merge( validationReportId, @@ -199,6 +200,10 @@ public void cleanUp(String validationReportId) { scheduledStopPointAndQuayIdCache.remove(validationReportId); serviceLinksAndFromToScheduledStopPointIdCache.remove(validationReportId); lineInfoCache.remove(validationReportId); + redissonClient.getKeys().deleteByPattern(validationReportId + '*'); + serviceJourneyStopsCache + .keySet() + .removeIf(k -> k.startsWith(validationReportId)); serviceJourneyStopsCache.remove(validationReportId); serviceJourneyInterchangeInfoCache.remove(validationReportId); } diff --git a/src/main/java/no/entur/antu/netexdata/collectors/ServiceJourneyStopsCollector.java b/src/main/java/no/entur/antu/netexdata/collectors/ServiceJourneyStopsCollector.java index 87719efb..bfbadede 100644 --- a/src/main/java/no/entur/antu/netexdata/collectors/ServiceJourneyStopsCollector.java +++ b/src/main/java/no/entur/antu/netexdata/collectors/ServiceJourneyStopsCollector.java @@ -9,6 +9,7 @@ import org.entur.netex.validation.validator.model.ScheduledStopPointId; import org.entur.netex.validation.validator.model.ServiceJourneyStop; import org.redisson.api.RLock; +import org.redisson.api.RMap; import org.redisson.api.RedissonClient; import org.springframework.stereotype.Component; @@ -44,7 +45,6 @@ protected void collectDataFromLineFile( Map> serviceJourneyStops = antuNetexData .validServiceJourneys() - // TODO: unique service journeys ids .map(serviceJourney -> { Map scheduledStopPointIdMap = AntuNetexData.scheduledStopPointIdByStopPointId( @@ -71,6 +71,7 @@ protected void collectDataFromLineFile( addServiceJourneyStops( validationContext.getValidationReportId(), + validationContext.getFileName(), serviceJourneyStops ); } @@ -84,19 +85,20 @@ protected void collectDataFromCommonFile( private void addServiceJourneyStops( String validationReportId, + String filename, Map> serviceJourneyStops ) { RLock lock = redissonClient.getLock(validationReportId); try { lock.lock(); - serviceJourneyStopsCache.merge( - validationReportId, - serviceJourneyStops, - (existingMap, newMap) -> { - existingMap.putAll(newMap); - return existingMap; - } + RMap> serviceJourneyStopsMap = redissonClient.getMap( + validationReportId + "_" + filename + ); + serviceJourneyStopsMap.putAll(serviceJourneyStops); + serviceJourneyStopsCache.put( + validationReportId + "_" + filename, + serviceJourneyStopsMap ); } finally { if (lock.isHeldByCurrentThread()) {