Merge pull request #3006 from squaregoldfish/2964

2964 RAM usage in DataSet processing

WebApp/WebContent/META-INF/context.xml

@@ -12,5 +12,5 @@
      maxAge="25000" testOnBorrow="true" testWhileIdle="true" validationInterval="0"
      removeAbandoned="true" logAbandoned="true" removeAbandonedTimeout="3600"
      username="%db_username%" password="%db_password%" driverClassName="com.mysql.jdbc.Driver"
-     url="jdbc:mysql://%db_host%:%db_port%/%db_database%?useUnicode=true&characterEncoding=utf-8&rewriteBatchedStatements=true" />
+     url="jdbc:mysql://%db_host%:%db_port%/%db_database%?useUnicode=true&characterEncoding=utf-8&useServerPrepStmts=false&rewriteBatchedStatements=true" />
 </Context>

WebApp/junit/resources/testsets/LatitudeTests.csv

@@ -19,10 +19,6 @@ FORMAT_0_90,0.5,N,true,0.500
 FORMAT_0_90,0.5,S,true,-0.500
 FORMAT_0_90,0.5,N,true,0.500
 FORMAT_0_90,0.5,S,true,-0.500
-FORMAT_0_90,0.5,North,true,0.500
-FORMAT_0_90,0.5,South,true,-0.500
-FORMAT_0_90,0.5,North,true,0.500
-FORMAT_0_90,0.5,South,true,-0.500
 FORMAT_0_90,60.342,N,true,60.342
 FORMAT_0_90,60.342,S,true,-60.342
 FORMAT_0_90,-1,N,false,

WebApp/junit/resources/testsets/LongitudeTests.csv

@@ -28,10 +28,6 @@ FORMAT_0_180,0.5,E,true,0.500
 FORMAT_0_180,0.5,W,true,-0.500
 FORMAT_0_180,0.5,e,true,0.500
 FORMAT_0_180,0.5,w,true,-0.500
-FORMAT_0_180,0.5,East,true,0.500
-FORMAT_0_180,0.5,West,true,-0.500
-FORMAT_0_180,0.5,east,true,0.500
-FORMAT_0_180,0.5,west,true,-0.500
 FORMAT_0_180,100.342,E,true,100.342
 FORMAT_0_180,100.342,W,true,-100.342
 FORMAT_0_180,-1,E,false,

WebApp/junit/uk/ac/exeter/QuinCe/data/Dataset/ControsPco2MeasurementLocatorTest.java

@@ -87,8 +87,11 @@ public void noZeroFlushingTimeTest() throws Exception {
 
     ControsPco2MeasurementLocator locator = new ControsPco2MeasurementLocator();
 
+    DatasetSensorValues sensorValues = DataSetDataDB.getSensorValues(
+      getConnection(), getInstrument(), getDataset().getId(), false, true);
+
     List<Measurement> measurements = locator.locateMeasurements(getConnection(),
-      getInstrument(), getDataset());
+      getInstrument(), getDataset(), sensorValues);
 
     List<LocalDateTime> locatedMeasurementTimes = measurements.stream()
       .map(m -> m.getTime()).toList();

WebApp/junit/uk/ac/exeter/QuinCe/data/Dataset/DataSetDataDBTest.java

@@ -84,9 +84,9 @@ public void storeSensorValuesSingleValueTest() throws Exception {
       Arrays.asList(sensorValue));
     SensorValue storedValue = retrieveSingleStoredValue(valueTime);
 
-    // Show that the SensorValue's ID has not been updated
-    assertEquals(DatabaseUtils.NO_DATABASE_RECORD, sensorValue.getId(),
-      "Sensor Value ID was updated unexpectedly");
+    // Show that the SensorValue's ID has been updated
+    assertNotEquals(DatabaseUtils.NO_DATABASE_RECORD, sensorValue.getId(),
+      "Sensor Value ID was not updated");
 
     // Check the dirty flag
     assertFalse(sensorValue.isDirty(), "Dirty flag not cleared");

WebApp/junit/uk/ac/exeter/QuinCe/data/Dataset/DatasetSensorValuesTest/QCCascade/AbstractDiagnosticFlagTest.java

@@ -133,7 +133,7 @@ protected DataReductionRecord runDataReduction(Connection conn,
     MeasurementLocator measurementLocator = new SimpleMeasurementLocator(
       variable);
     List<Measurement> locatedMeasurements = measurementLocator
-      .locateMeasurements(conn, instrument, dataset);
+      .locateMeasurements(conn, instrument, dataset, allSensorValues);
 
     DataSetDataDB.storeMeasurements(conn, locatedMeasurements);
 

WebApp/junit/uk/ac/exeter/QuinCe/data/Dataset/DatasetSensorValuesTest/QCCascade/FlagCascadeTest.java

@@ -117,7 +117,7 @@ public void flagCascadeTest(TestSetLine line) throws Exception {
 
       // Set the QC flags on sensor values
       DatasetSensorValues allSensorValues = DataSetDataDB.getSensorValues(conn,
-        instrument, dataset.getId(), false, false);
+        instrument, dataset.getId(), false, true);
 
       SensorValue sstVal = allSensorValues.getById(2L);
       SensorValue salVal = allSensorValues.getById(3L);
@@ -132,7 +132,7 @@ public void flagCascadeTest(TestSetLine line) throws Exception {
       MeasurementLocator measurementLocator = new SimpleMeasurementLocator(
         variable);
       List<Measurement> locatedMeasurements = measurementLocator
-        .locateMeasurements(conn, instrument, dataset);
+        .locateMeasurements(conn, instrument, dataset, allSensorValues);
 
       DataSetDataDB.storeMeasurements(conn, locatedMeasurements);
 

WebApp/junit/uk/ac/exeter/QuinCe/data/Instrument/DataFormats/HemisphereMultiplierTest.java

@@ -3,100 +3,62 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-
 import org.junit.jupiter.api.Test;
 
 import uk.ac.exeter.QuinCe.TestBase.BaseTest;
-import uk.ac.exeter.QuinCe.utils.MissingParamException;
 
 public class HemisphereMultiplierTest extends BaseTest {
 
-  @Test
-  public void nullOneConstructorTest() {
-    assertThrows(MissingParamException.class, () -> {
-      new HemisphereMultiplier(null, Arrays.asList("S", "South"));
-    });
-  }
-
-  @Test
-  public void emptyOneConstructorTest() {
-    assertThrows(MissingParamException.class, () -> {
-      new HemisphereMultiplier(new ArrayList<String>(),
-        Arrays.asList("S", "South"));
-    });
-  }
-
-  @Test
-  public void nullMinusOneConstructorTest() {
-    assertThrows(MissingParamException.class, () -> {
-      new HemisphereMultiplier(Arrays.asList("N", "North"), null);
-    });
-  }
-
-  @Test
-  public void emptyMinusOneConstructorTest() {
-    assertThrows(MissingParamException.class, () -> {
-      new HemisphereMultiplier(new ArrayList<String>(),
-        new ArrayList<String>());
-    });
-  }
-
   private double applyMultiplier(double input, String hemisphere)
     throws InvalidHemisphereException {
-    HemisphereMultiplier m = new HemisphereMultiplier(
-      Arrays.asList("N", "North"), Arrays.asList("S", "South"));
-    return m.apply(input, hemisphere);
 
+    return HemisphereMultiplier.apply(input, hemisphere);
   }
 
   @Test
-  public void firstOneEntryCaseMatch() throws InvalidHemisphereException {
+  public void test_N() throws InvalidHemisphereException {
     assertEquals(1D, applyMultiplier(1D, "N"));
   }
 
   @Test
-  public void secondOneEntryCaseMatch() throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(1D, "North"));
+  public void test_n() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(1D, "n"));
   }
 
   @Test
-  public void firstOneEntryCaseNotMatch() throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(1D, "n"));
+  public void test_S() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(-1D, "S"));
   }
 
   @Test
-  public void secondOneEntryCaseNotMatch() throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(1D, "nOrTH"));
+  public void test_s() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(-1D, "s"));
   }
 
   @Test
-  public void firstMinusOneEntryCaseMatch() throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(-1D, "S"));
+  public void test_E() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(1D, "E"));
   }
 
   @Test
-  public void secondMinusOneEntryCaseMatch() throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(-1D, "South"));
+  public void test_e() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(1D, "e"));
   }
 
   @Test
-  public void firstMinusOneEntryCaseNotMatch()
-    throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(-1D, "s"));
+  public void test_W() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(-1D, "W"));
   }
 
   @Test
-  public void secondMinusOneEntryCaseNotMatch()
-    throws InvalidHemisphereException {
-    assertEquals(1D, applyMultiplier(-1D, "SouTh"));
+  public void test_w() throws InvalidHemisphereException {
+    assertEquals(1D, applyMultiplier(-1D, "w"));
   }
 
   @Test
-  public void noMatchTest() {
+  public void testInvalid() {
     assertThrows(InvalidHemisphereException.class, () -> {
-      applyMultiplier(1D, "West");
+      applyMultiplier(1D, "fdljg");
     });
   }
 }