Allu 85: check for anonymizable cable reports

backend/external-service/src/main/java/fi/hel/allu/external/controller/maintenance/ApplicationArchiveController.java

@@ -45,4 +45,10 @@ public ResponseEntity<Void> archiveFinishedApplications(@RequestBody List<Intege
         return new ResponseEntity<>(HttpStatus.OK);
     }
 
+    @PatchMapping(value = "/checkanonymizable")
+    @PreAuthorize("hasAnyRole('ROLE_SERVICE')")
+    public ResponseEntity<Void> checkForAnonymizableApplications() {
+      applicationArchiverService.checkForAnonymizableApplications();
+      return new ResponseEntity<>(HttpStatus.OK);
+    }
 }

backend/model-service/src/main/java/fi/hel/allu/model/controller/ApplicationController.java

@@ -412,6 +412,17 @@ public ResponseEntity<List<Application>> findActiveExcavationAnnouncements() {
     return new ResponseEntity<>(applicationService.findActiveExcavationAnnouncements(), HttpStatus.OK);
   }
 
+  @GetMapping(value = "/potentiallyanonymizable")
+  public ResponseEntity<List<Application>> findPotentiallyAnonymizableApplications() {
+    return new ResponseEntity<>(applicationService.findPotentiallyAnonymizableApplications(), HttpStatus.OK);
+  }
+
+  @PatchMapping(value = "/addtoanonymizable")
+  public ResponseEntity<List<Application>> addToAnonymizableApplications(@RequestBody List<Integer> applicationsIds) {
+    applicationService.addToAnonymizableApplications(applicationsIds);
+    return new ResponseEntity<>(HttpStatus.OK);
+  }
+
   /**
    * Finds finished notes
    */

backend/model-service/src/main/java/fi/hel/allu/model/dao/ApplicationDao.java

@@ -9,6 +9,7 @@
 import com.querydsl.sql.SQLExpressions;
 import com.querydsl.sql.SQLQueryFactory;
 import com.querydsl.sql.dml.SQLInsertClause;
+import fi.hel.allu.QAnonymizableApplication;
 import fi.hel.allu.QApplication;
 import fi.hel.allu.common.domain.ApplicationDateReport;
 import fi.hel.allu.common.domain.ApplicationStatusInfo;
@@ -35,6 +36,7 @@
 import static com.querydsl.core.group.GroupBy.list;
 import static com.querydsl.core.types.Projections.bean;
 import static com.querydsl.sql.SQLExpressions.select;
+import static fi.hel.allu.QAnonymizableApplication.anonymizableApplication;
 import static fi.hel.allu.QApplication.application;
 import static fi.hel.allu.QApplicationCustomer.applicationCustomer;
 import static fi.hel.allu.QApplicationCustomerContact.applicationCustomerContact;
@@ -188,6 +190,22 @@ public List<Application> findActiveExcavationAnnouncements() {
     return queryFactory.select(applicationBean).from(application).where(whereCondition).fetch();
   }
 
+  @Transactional(readOnly = true)
+  public List<Application> fetchPotentiallyAnonymizableApplications() {
+    SubQueryExpression<Integer> alreadyFounds = select(anonymizableApplication.applicationId).from(anonymizableApplication);
+    BooleanExpression whereCondition = application.type.eq(ApplicationType.CABLE_REPORT);
+    whereCondition = whereCondition.and(application.endTime.before(ZonedDateTime.now().minusYears(2)));
+    whereCondition = whereCondition.and(application.id.notIn(alreadyFounds));
+    return queryFactory.select(applicationBean).from(application).where(whereCondition).fetch();
+  }
+
+  @Transactional
+  public void insertToAnonymizableApplication(List<Integer> applicationIds) {
+    SQLInsertClause insertQuery = queryFactory.insert(anonymizableApplication);
+    for (Integer applicationId : applicationIds) insertQuery.set(anonymizableApplication.applicationId, applicationId).addBatch();
+    insertQuery.execute();
+  }
+
   @Transactional(readOnly = true)
   public List<Integer> findFinishedNotes() {
     ZonedDateTime now = ZonedDateTime.now();

backend/model-service/src/main/java/fi/hel/allu/model/service/ApplicationService.java

@@ -429,6 +429,14 @@ public List<Application> findActiveExcavationAnnouncements() {
     return applicationDao.findActiveExcavationAnnouncements();
   }
 
+  public List<Application> findPotentiallyAnonymizableApplications() {
+    return applicationDao.fetchPotentiallyAnonymizableApplications();
+  }
+
+  public void addToAnonymizableApplications(List<Integer> applicationsIds) {
+    applicationDao.insertToAnonymizableApplication(applicationsIds);
+  }
+
   @Transactional(readOnly = true)
   public List<Integer> findFinishedNotes() {
     return applicationDao.findFinishedNotes();

backend/model-service/src/test/java/fi/hel/allu/model/dao/ApplicationDaoTest.java

@@ -1,5 +1,7 @@
 package fi.hel.allu.model.dao;
 
+import com.querydsl.sql.SQLQueryFactory;
+import fi.hel.allu.QAnonymizableApplication;
 import fi.hel.allu.common.domain.types.*;
 import fi.hel.allu.common.types.DistributionType;
 import fi.hel.allu.model.ModelApplication;
@@ -20,6 +22,7 @@
 import java.time.ZonedDateTime;
 import java.util.*;
 
+import static fi.hel.allu.QAnonymizableApplication.anonymizableApplication;
 import static org.junit.Assert.*;
 
 @RunWith(SpringJUnit4ClassRunner.class)
@@ -37,6 +40,9 @@ public class ApplicationDaoTest {
   @Autowired
   private DistributionEntryDao distributionEntryDao;
 
+  @Autowired
+  private SQLQueryFactory queryFactory;
+
   DistributionEntry testDistributionEntry;
 
   @Before
@@ -398,6 +404,59 @@ public void testFindActiveExcavationAnnouncements() {
     }
   }
 
+  @Test
+  public void testFetchPotentiallyAnonymizableApplications() {
+
+    // wrong types of applications
+    Application outdoor = testCommon.dummyOutdoorApplication("Outdoor application", "Outsider");
+    outdoor.setStatus(StatusType.HANDLING);
+    outdoor = applicationDao.insert(outdoor);
+
+    Application areaRental = testCommon.dummyAreaRentalApplication("Area rental", "Renter");
+    areaRental.setStatus(StatusType.DECISION);
+    areaRental = applicationDao.insert(areaRental);
+
+    // long enough ago and not already found - we want to get this one from the query
+    Application cableReport1 = testCommon.dummyCableReportApplication("Cable report1", "Reporter1");
+    cableReport1.setStatus(StatusType.DECISION);
+    cableReport1.setEndTime(ZonedDateTime.now().minusYears(2).minusMonths(6));
+    cableReport1 = applicationDao.insert(cableReport1);
+
+    // not long enough ago
+    Application cableReport2 = testCommon.dummyCableReportApplication("Cable report2", "Reporter2");
+    cableReport2.setStatus(StatusType.DECISION);
+    cableReport2.setEndTime(ZonedDateTime.now().minusYears(1).minusMonths(6));
+    cableReport2 = applicationDao.insert(cableReport2);
+
+    // long enough ago but already found
+    Application cableReport3 = testCommon.dummyCableReportApplication("Cable report3", "Reporter3");
+    cableReport3.setStatus(StatusType.DECISION);
+    cableReport3.setEndTime(ZonedDateTime.now().minusYears(2).minusMonths(6));
+    cableReport3 = applicationDao.insert(cableReport3);
+    applicationDao.insertToAnonymizableApplication(List.of(cableReport3.getId()));
+
+    List<Application> potentials = applicationDao.fetchPotentiallyAnonymizableApplications();
+
+    assertEquals(1, potentials.size());
+    assertEquals(potentials.get(0).getId(), cableReport1.getId());
+  }
+
+  @Test
+  public void testInsertToAnonymizableApplication() {
+    Integer app1 = testCommon.insertApplication("Outdoor1", "Customer1");
+    Integer app2 = testCommon.insertApplication("Outdoor2", "Customer2");
+    Integer app3 = testCommon.insertApplication("Outdoor3", "Customer3");
+
+    applicationDao.insertToAnonymizableApplication(List.of(app1, app2, app3));
+
+    List<Integer> applicationIds = queryFactory.select(anonymizableApplication.applicationId).from(anonymizableApplication).fetch();
+
+    assertEquals(3, applicationIds.size());
+    assert(applicationIds.contains(app1));
+    assert(applicationIds.contains(app2));
+    assert(applicationIds.contains(app3));
+  }
+
   private ApplicationTag createApplicationTag(ApplicationTagType applicationTagType) {
     ApplicationTag applicationTag = new ApplicationTag();
     applicationTag.setAddedBy(1);

backend/model-service/src/test/java/fi/hel/allu/model/testUtils/TestCommon.java

@@ -192,6 +192,14 @@ public Application dummyExcavationAnnouncementApplication(String name, String ow
     return app;
   }
 
+  public Application dummyCableReportApplication(String name, String owner) {
+    Application app = dummyBasicApplication(name, owner);
+    app.setType(ApplicationType.CABLE_REPORT);
+    app.setKindsWithSpecifiers(Collections.singletonMap(ApplicationKind.STREET_AND_GREEN, Collections.emptyList()));
+    app.setExtension(dummyCableReport());
+    return app;
+  }
+
   /**
    * Create a dummy outdoor applicationExtension.
    *
@@ -237,6 +245,11 @@ public ApplicationExtension dummyExcavationAnnouncement() {
     return excavationAnnouncement;
   }
 
+  public ApplicationExtension dummyCableReport() {
+    CableReport cableReport = new CableReport();
+    return cableReport;
+  }
+
   public ApplicationTag dummyTag(ApplicationTagType tagType) {
     ApplicationTag tag = new ApplicationTag();
     tag.setType(tagType);

backend/scheduler-service/src/main/java/fi/hel/allu/scheduler/config/ApplicationProperties.java

@@ -395,6 +395,10 @@ public String getUpdateTerminatedApplicationsUrl() {
     return getExtServiceUrl("/v1/applications/terminated/status");
   }
 
+  public String getCheckAnonymizableApplicationsUrl() {
+    return getExtServiceUrl("/v1/applications/checkanonymizable");
+  }
+
   public String getKnownHosts() {
     return knownHosts;
   }
@@ -410,4 +414,4 @@ public String getKeyAlgorithm() {
   public int getSftpTimeout() {
     return sftpTimeout;
   }
-}
+}

backend/scheduler-service/src/main/java/fi/hel/allu/scheduler/controller/ScheduleRunner.java

@@ -104,4 +104,9 @@ public void updateApplicationStatuses() {
   public void updateCityDistricts() {
     cityDistrictUpdaterService.updateCityDistricts();
   }
+
+  @Scheduled(cron = "${anonymization.update.cronstring}")
+  public void checkAnonymizableApplications() {
+    applicationStatusUpdaterService.checkAnonymizableApplications();
+  }
 }

backend/scheduler-service/src/main/java/fi/hel/allu/scheduler/service/ApplicationStatusUpdaterService.java

@@ -56,4 +56,11 @@ public void archiveApplicationStatusesForTerminated() {
         applicationProperties.getUpdateTerminatedApplicationsUrl(), HttpMethod.PATCH,
         new HttpEntity<>(null, authenticationService.createAuthenticationHeader()), Void.class).getBody();
   }
+
+  public void checkAnonymizableApplications() {
+    logger.info("Checking for anonymizable applications");
+    restTemplate.exchange(
+      applicationProperties.getCheckAnonymizableApplicationsUrl(), HttpMethod.PATCH,
+      new HttpEntity<>(null, authenticationService.createAuthenticationHeader()), Void.class).getBody();
+  }
 }

backend/scheduler-service/src/main/resources/application.properties

@@ -76,4 +76,6 @@ invoice.notification.subject=Allu - uusia laskuja
 
 application.status.update.cronstring=0 1 * * * *
 
-cityDistricts.update.cronstring=30 0 * * * *
+cityDistricts.update.cronstring=30 0 * * * *
+
+anonymization.update.cronstring=0 */5 * * * *

backend/service-core/pom.xml

@@ -96,6 +96,10 @@
           <artifactId>jwks-rsa</artifactId>
           <version>0.21.1</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
         <!-- Mock web server dependencies -->
         <dependency>
           <groupId>com.squareup.okhttp3</groupId>
@@ -121,4 +125,4 @@
             <scope>runtime</scope>
         </dependency>
     </dependencies>
-</project>
+</project>

backend/service-core/src/main/java/fi/hel/allu/servicecore/config/ApplicationProperties.java

@@ -1396,6 +1396,14 @@ public String getActiveExcavationAnnouncementsUrl() {
     return getModelServiceUrl("/applications/activeexcavationannouncements");
   }
 
+  public String getFetchPotentiallyAnonymizableApplicationsUrl() {
+    return getModelServiceUrl("/applications/potentiallyanonymizable");
+  }
+
+  public String getAddToAnonymizableApplicationsUrl() {
+    return getModelServiceUrl("/applications/addtoanonymizable");
+  }
+
   /**
    * @return url to fetch finished notes
    */

backend/service-core/src/main/java/fi/hel/allu/servicecore/service/ApplicationArchiverService.java

@@ -2,11 +2,13 @@
 
 import java.time.ZonedDateTime;
 import java.util.*;
+import java.util.stream.Collectors;
 
 import fi.hel.allu.common.domain.TerminationInfo;
 import fi.hel.allu.common.domain.types.*;
 import fi.hel.allu.common.util.TimeUtil;
 import fi.hel.allu.model.domain.Application;
+import fi.hel.allu.model.domain.CableReport;
 import fi.hel.allu.model.domain.ExcavationAnnouncement;
 import fi.hel.allu.servicecore.domain.CableReportJson;
 import fi.hel.allu.servicecore.domain.ExcavationAnnouncementJson;
@@ -108,6 +110,14 @@ private List<Application> fetchActiveExcavationAnnouncements() {
     return applicationServiceComposer.fetchActiveExcavationAnnouncements();
   }
 
+  private List<Application> fetchPotentiallyAnonymizableApplications() {
+    return applicationServiceComposer.fetchPotentiallyAnonymizableApplications();
+  }
+
+  private void addToAnonymizableApplications(List<Integer> applicationIds) {
+    applicationServiceComposer.addToAnonymizableApplications(applicationIds);
+  }
+
   /**
    * Archives applications if necessary.
    * See {@link #archiveApplicationIfNecessary(Integer)}
@@ -225,16 +235,41 @@ private boolean hasOpenSupervisionTask(int applicationId, SupervisionTaskType ta
         && taskType.equals(t.getType()));
   }
 
+  private boolean cableReportAssociatedWithActiveExcavationAnnouncement(Application application) {
+    return cableReportAssociatedWithActiveExcavationAnnouncement(application.getType(), application.getApplicationId());
+  }
+
   private boolean cableReportAssociatedWithActiveExcavationAnnouncement(ApplicationJson application) {
-    if (application.getType() != ApplicationType.CABLE_REPORT) return false;
+    return cableReportAssociatedWithActiveExcavationAnnouncement(application.getType(), application.getApplicationId());
+  }
+
+  private boolean cableReportAssociatedWithActiveExcavationAnnouncement(ApplicationType applicationType, String applicationId) {
+    if (applicationType != ApplicationType.CABLE_REPORT) return false;
     if (activeExcavationAnnouncements == null) activeExcavationAnnouncements = fetchActiveExcavationAnnouncements();
     for (Application excavationAnnouncement : activeExcavationAnnouncements) {
       if (excavationAnnouncement.getExtension() != null && excavationAnnouncement.getExtension() instanceof ExcavationAnnouncement extension) {
-          if (extension.getCableReports() != null && extension.getCableReports().contains(application.getApplicationId())) {
+          if (extension.getCableReports() != null && extension.getCableReports().contains(applicationId)) {
             return true;
           }
       }
     }
     return false;
   }
+  public void checkForAnonymizableApplications() {
+    List<Integer> anonymizeIds =
+      fetchPotentiallyAnonymizableApplications().stream()
+      // cable reports are all we know how to handle for now, let's be sure although DB should return only them currently
+      .filter(app -> app.getType() == ApplicationType.CABLE_REPORT)
+      .filter(app -> app.getExtension() != null)
+      .filter(app -> app.getExtension() instanceof CableReport)
+      .filter(app -> ((CableReport) app.getExtension()).getValidityTime().isBefore(ZonedDateTime.now().minusYears(2)))
+      .filter(app -> !cableReportAssociatedWithActiveExcavationAnnouncement(app))
+      .map(Application::getId)
+      .toList();
+
+    if (anonymizeIds.size() > 0) addToAnonymizableApplications(anonymizeIds);
+
+    // these might have been fetched in cableReportAssociatedWithActiveExcavationAnnouncement(), release them now
+    activeExcavationAnnouncements = null;
+  }
 }

backend/service-core/src/main/java/fi/hel/allu/servicecore/service/ApplicationService.java

@@ -21,6 +21,7 @@
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpMethod;
 import org.springframework.http.ResponseEntity;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
 import org.springframework.stereotype.Service;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponentsBuilder;
@@ -57,6 +58,9 @@ public ApplicationService(
       ApplicationEventDispatcher applicationEventDispatcher) {
     this.applicationProperties = applicationProperties;
     this.restTemplate = restTemplate;
+    // Needed for PATCH support
+    HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory();
+    restTemplate.setRequestFactory(requestFactory);
     this.applicationMapper = applicationMapper;
     this.userService = userService;
     this.personAuditLogService = personAuditLogService;
@@ -395,6 +399,17 @@ public List<Application> findActiveExcavationAnnouncements() {
       HttpMethod.GET, null, typeRef).getBody();
   }
 
+  public List<Application> fetchPotentiallyAnonymizableApplications() {
+    ParameterizedTypeReference<List<Application>> typeRef = new ParameterizedTypeReference<List<Application>>() {};
+    return restTemplate.exchange(applicationProperties.getFetchPotentiallyAnonymizableApplicationsUrl(),
+      HttpMethod.GET, null, typeRef).getBody();
+  }
+
+  public void addToAnonymizableApplications(List<Integer> applicationIds) {
+    restTemplate.exchange(applicationProperties.getAddToAnonymizableApplicationsUrl(),
+      HttpMethod.PATCH, new HttpEntity<>(applicationIds), Void.class);
+  }
+
   public List<Integer> findFinishedNotes() {
     ParameterizedTypeReference<List<Integer>> typeRef = new ParameterizedTypeReference<List<Integer>>() {};
     return restTemplate.exchange(applicationProperties.getFinishedNotesUrl(), HttpMethod.GET, null, typeRef).getBody();