(#2898) Handle missing flushing flags

Simple algorithm: If there's no flushing flags after a zero, flush the next records within the period `Response Time * 7`
quince-science · May 16, 2024 · f30af89 · f30af89
1 parent 36a5f9d
commit f30af89
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 6 deletions.
diff --git a/WebApp/junit/junit/uk/ac/exeter/QuinCe/data/Dataset/ControsPco2MeasurementLocatorTest.java b/WebApp/junit/junit/uk/ac/exeter/QuinCe/data/Dataset/ControsPco2MeasurementLocatorTest.java
@@ -116,9 +116,6 @@ private List<LocalDateTime> getNoZeroExpectedMeasurementTimes() {
     times.add(LocalDateTime.of(2023, 6, 8, 18, 10, 53));
     times.add(LocalDateTime.of(2023, 6, 8, 18, 11, 3));
     times.add(LocalDateTime.of(2023, 6, 8, 18, 11, 13));
-    times.add(LocalDateTime.of(2023, 6, 8, 23, 50, 13));
-    times.add(LocalDateTime.of(2023, 6, 8, 23, 51, 13));
-    times.add(LocalDateTime.of(2023, 6, 8, 23, 52, 13));
     times.add(LocalDateTime.of(2023, 6, 8, 23, 53, 13));
     times.add(LocalDateTime.of(2023, 6, 8, 23, 54, 13));
     times.add(LocalDateTime.of(2023, 6, 8, 23, 55, 13));

diff --git a/WebApp/junit/resources/db_migrations/V42__contros_response_time_2898.sql b/WebApp/junit/resources/db_migrations/V42__contros_response_time_2898.sql
@@ -0,0 +1,2 @@
+-- Add Response Time to coefficients
+UPDATE variables SET properties = '{"coefficients": ["F", "Runtime", "k1", "k2", "k3", "Response Time"]}' WHERE id = 6;
diff --git a/...taReduction/ControsPco2ReducerTest/pre-calibration/V2000001__contros_pre-calibrations.sql b/...taReduction/ControsPco2ReducerTest/pre-calibration/V2000001__contros_pre-calibrations.sql
@@ -16,3 +16,6 @@ INSERT INTO calibration (instrument_id, type, target, deployment_date, coefficie
 
 INSERT INTO calibration (instrument_id, type, target, deployment_date, coefficients, class)
   VALUES (124, 'CALC_COEFFICIENT', '6.k3', 1626393600000, '{"Value":"2.965241e-10"}', 'CalculationCoefficient');
+
+  INSERT INTO calibration (instrument_id, type, target, deployment_date, coefficients, class)
+  VALUES (124, 'CALC_COEFFICIENT', '6.Response Time', 1626393600000, '{"Value":"25"}', 'CalculationCoefficient');
diff --git a/WebApp/src/uk/ac/exeter/QuinCe/data/Dataset/ControsPco2MeasurementLocator.java b/WebApp/src/uk/ac/exeter/QuinCe/data/Dataset/ControsPco2MeasurementLocator.java
@@ -9,6 +9,9 @@
 
 import uk.ac.exeter.QuinCe.data.Dataset.QC.Flag;
 import uk.ac.exeter.QuinCe.data.Instrument.Instrument;
+import uk.ac.exeter.QuinCe.data.Instrument.Calibration.CalculationCoefficient;
+import uk.ac.exeter.QuinCe.data.Instrument.Calibration.CalculationCoefficientDB;
+import uk.ac.exeter.QuinCe.data.Instrument.Calibration.CalibrationSet;
 import uk.ac.exeter.QuinCe.data.Instrument.SensorDefinition.SensorType;
 import uk.ac.exeter.QuinCe.data.Instrument.SensorDefinition.SensorsConfiguration;
 import uk.ac.exeter.QuinCe.data.Instrument.SensorDefinition.Variable;
@@ -25,6 +28,8 @@ public class ControsPco2MeasurementLocator extends MeasurementLocator {
 
   private static final int MEASUREMENT = 2;
 
+  private static final int RESPONSE_TIME_MULTIPLIER = 7;
+
   @Override
   public List<Measurement> locateMeasurements(Connection conn,
     Instrument instrument, DataSet dataset) throws MeasurementLocatorException {
@@ -57,13 +62,22 @@ public List<Measurement> locateMeasurements(Connection conn,
       DatasetSensorValues sensorValues = DataSetDataDB.getSensorValues(conn,
         instrument, dataset.getId(), false, true);
 
+      CalibrationSet priorCoefficients = CalculationCoefficientDB.getInstance()
+        .getMostRecentCalibrations(conn, instrument,
+          sensorValues.getTimes().get(0));
+
+      long defaultFlushingTime = Math.round(CalculationCoefficient
+        .getCoefficient(priorCoefficients, variable, "Response Time")
+        .getValue()) * RESPONSE_TIME_MULTIPLIER;
+
       // Loop through all the rows, examining the zero/flush columns to decide
       // what to do
       List<SensorValue> flaggedSensorValues = new ArrayList<SensorValue>();
       List<Measurement> measurements = new ArrayList<Measurement>(
         sensorValues.getTimes().size());
 
       int currentStatus = NO_STATUS;
+      int lastStatus = NO_STATUS;
       LocalDateTime currentStatusStart = null;
 
       for (LocalDateTime recordTime : sensorValues.getTimes()) {
@@ -77,6 +91,7 @@ public List<Measurement> locateMeasurements(Connection conn,
             flushingColumn);
 
           if (recordStatus != currentStatus) {
+            lastStatus = currentStatus;
             currentStatus = recordStatus;
             currentStatusStart = recordTime;
           }
@@ -95,6 +110,15 @@ public List<Measurement> locateMeasurements(Connection conn,
           } else {
             if (recordStatus == FLUSHING) {
               flushSensors = true;
+            } else {
+              // If there hasn't been a FLUSHING flag since the last zero,
+              // flush for the default period calculated from the response time.
+              if (lastStatus == ZERO
+                && DateTimeUtils.secondsBetween(currentStatusStart,
+                  recordTime) <= defaultFlushingTime) {
+
+                flushSensors = true;
+              }
             }
 
             runType = Measurement.MEASUREMENT_RUN_TYPE;

diff --git a/WebApp/src/uk/ac/exeter/QuinCe/data/Instrument/Calibration/CalculationCoefficientDB.java b/WebApp/src/uk/ac/exeter/QuinCe/data/Instrument/Calibration/CalculationCoefficientDB.java
@@ -1,9 +1,9 @@
 package uk.ac.exeter.QuinCe.data.Instrument.Calibration;
 
 import java.sql.Connection;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
 import java.util.stream.Collectors;
 
 import uk.ac.exeter.QuinCe.data.Instrument.Instrument;
@@ -31,7 +31,7 @@ public Map<String, String> getTargets(Connection conn, Instrument instrument)
     // If there's only one variable, leave the variable name off the
     // human-readable side
 
-    Map<String, String> targets = new TreeMap<String, String>();
+    Map<String, String> targets = new LinkedHashMap<String, String>();
 
     List<Variable> variablesWithCoefficients = instrument.getVariables()
       .stream().filter(v -> v.hasCoefficients()).collect(Collectors.toList());

diff --git a/WebApp/src/uk/ac/exeter/QuinCe/web/files/MultipleFileUploadBean.java b/WebApp/src/uk/ac/exeter/QuinCe/web/files/MultipleFileUploadBean.java
@@ -48,7 +48,8 @@ public TreeSet<UploadedDataFile> getUploadedFiles() {
 
   public List<RunTypeAssignment> getAllRunTypes() {
     return getCurrentInstrument().getFileDefinitions().stream()
-      .map(df -> df.getRunTypes().values()).flatMap(Collection::stream)
+      .filter(fd -> fd.getRunTypes() != null)
+      .map(fd -> fd.getRunTypes().values()).flatMap(Collection::stream)
       .distinct().collect(Collectors.toList());
   }
 

diff --git a/src/migrations/db_migrations/V42__contros_response_time_2898.sql b/src/migrations/db_migrations/V42__contros_response_time_2898.sql
@@ -0,0 +1,2 @@
+-- Add Response Time to coefficients
+UPDATE variables SET properties = '{"coefficients": ["F", "Runtime", "k1", "k2", "k3", "Response Time"]}' WHERE id = 6;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		-- Add Response Time to coefficients
		UPDATE variables SET properties = '{"coefficients": ["F", "Runtime", "k1", "k2", "k3", "Response Time"]}' WHERE id = 6;