Skip to content

Commit

Permalink
(#2906) No NRT export within 24h of previous
Browse files Browse the repository at this point in the history
  • Loading branch information
squaregoldfish committed Jul 5, 2024
1 parent 7fcb42d commit 2fb1b81
Show file tree
Hide file tree
Showing 16 changed files with 153 additions and 50 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
-- Add field for last NRT export time
ALTER TABLE instrument ADD COLUMN last_nrt_export BIGINT NULL DEFAULT NULL AFTER `nrt`;
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
-- It consists of a sequence of ZERO, FLUSH, MEASUREMENTS, ZERO

-- Instrument
INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,NULL,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');

-- Instrument Variables
INSERT INTO `instrument_variables` VALUES (124,6,'{"zero_flush":"0.00","zero_mode":"Continuous"}','2024-02-01 15:01:57','2024-02-01 16:01:57');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
-- There is no pre-zero

-- Instrument
INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,NULL,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');

-- Instrument Variables
INSERT INTO `instrument_variables` VALUES (124,6,'{"zero_flush":"0.00","zero_mode":"Continuous"}','2024-02-01 15:01:57','2024-02-01 16:01:57');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
-- There is no post-zerp

-- Instrument
INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,NULL,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');

-- Instrument Variables
INSERT INTO `instrument_variables` VALUES (124,6,'{"zero_flush":"0.00","zero_mode":"Continuous"}','2024-02-01 15:01:57','2024-02-01 16:01:57');
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ INSERT INTO variable_sensors


-- Now an instrument using the variable
INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,NULL,'','2019-01-28 13:31:21','2019-01-28 14:31:21');

INSERT INTO instrument_variables (instrument_id, variable_id)
VALUES (1, (SELECT id FROM variables WHERE name = 'testVar'));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ INSERT INTO variable_sensors


-- Now an instrument using the variable
INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,NULL,'','2019-01-28 13:31:21','2019-01-28 14:31:21');

INSERT INTO instrument_variables (instrument_id, variable_id)
VALUES (1, (SELECT id FROM variables WHERE name = 'testVar'));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
-- Benguela Stream instrument definition

-- Instrument
INSERT INTO instrument VALUES (1,1,'Benguela Stream','Benguela Stream','BSBS',0,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
INSERT INTO instrument VALUES (1,1,'Benguela Stream','Benguela Stream','BSBS',0,NULL,'','2019-01-28 13:31:21','2019-01-28 14:31:21');

-- Instrument uses the basic marine pCO2 variable
INSERT INTO instrument_variables (instrument_id, variable_id)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,6 @@ protected int getNewStatus() {

@Override
protected void additionalAction(Connection conn, long id) throws Exception {
DataSetDB.setDatasetExported(conn, id);
DataSetDB.setDatasetExported(conn, id, true);
}
}
44 changes: 33 additions & 11 deletions WebApp/src/uk/ac/exeter/QuinCe/api/export/ExportList.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package uk.ac.exeter.QuinCe.api.export;

import java.sql.Connection;
import java.time.LocalDateTime;
import java.util.Collection;

import javax.ws.rs.GET;
Expand All @@ -16,6 +17,7 @@
import uk.ac.exeter.QuinCe.data.Instrument.Instrument;
import uk.ac.exeter.QuinCe.data.Instrument.InstrumentDB;
import uk.ac.exeter.QuinCe.utils.DatabaseUtils;
import uk.ac.exeter.QuinCe.utils.DateTimeUtils;
import uk.ac.exeter.QuinCe.utils.ExceptionUtils;
import uk.ac.exeter.QuinCe.web.system.ResourceManager;

Expand Down Expand Up @@ -44,6 +46,8 @@
@Path("/export/exportList")
public class ExportList {

private static final long MIN_NRT_EXPORT_INTERVAL = 86400L;

/**
* The main processing method for the API call.
*
Expand Down Expand Up @@ -71,22 +75,40 @@ public String getExportList() throws Exception {
JsonArray json = new JsonArray();

for (DataSet dataset : datasets) {
JsonObject datasetJson = new JsonObject();

datasetJson.addProperty("id", dataset.getId());
datasetJson.addProperty("name", dataset.getName());

Instrument instrument = InstrumentDB.getInstrument(conn,
dataset.getInstrumentId());
JsonObject instrumentJson = new JsonObject();
instrumentJson.addProperty("name", instrument.getName());
instrumentJson.addProperty("user", instrument.getOwner().getFullName());
datasetJson.add("instrument", instrumentJson);

json.add(datasetJson);
boolean canExport = true;

if (dataset.isNrt()) {
if (null != instrument.getLastNrtExport()) {
long timeSinceLastNrtExport = DateTimeUtils.secondsBetween(
instrument.getLastNrtExport(), LocalDateTime.now());
if (timeSinceLastNrtExport < MIN_NRT_EXPORT_INTERVAL) {
canExport = false;
}
}
}

if (canExport) {
JsonObject datasetJson = new JsonObject();

datasetJson.addProperty("id", dataset.getId());
datasetJson.addProperty("name", dataset.getName());

JsonObject instrumentJson = new JsonObject();
instrumentJson.addProperty("name", instrument.getName());
instrumentJson.addProperty("user",
instrument.getOwner().getFullName());
datasetJson.add("instrument", instrumentJson);

json.add(datasetJson);
}
}
result = json.toString();
} catch (Exception e) {
} catch (

Exception e) {
ExceptionUtils.printStackTrace(e);
throw e;
} finally {
Expand Down
59 changes: 53 additions & 6 deletions WebApp/src/uk/ac/exeter/QuinCe/data/Dataset/DataSetDB.java
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ public class DataSetDB {
private static final String DATASET_EXPORTED_STATEMENT = "UPDATE dataset "
+ "SET exported = 1 WHERE id = ?";

private static final String LAST_NRT_STATEMENT = "UPDATE instrument SET "
+ "last_nrt_export = ? WHERE id = ?";

private static final String SENSOR_OFFSETS_PROPERTY = "__SENSOR_OFFSETS";

/**
Expand Down Expand Up @@ -1261,18 +1264,62 @@ public static Map<Long, Integer> getDataSetCounts(DataSource dataSource,
return result;
}

public static void setDatasetExported(Connection conn, long datasetId)
throws DatabaseException {
/**
* Mark a {@link DataSet} as having been exported.
*
* @param conn
* A database connection.
* @param datasetId
* The {@link DataSet}'s ID
* @param recordNrt
* If this is an NRT dataset, record the last NRT export time for the
* instrument.
* @throws DatabaseException
* @throws RecordNotFoundException
* @throws MissingParamException
*/
public static void setDatasetExported(Connection conn, long datasetId,
boolean recordNrt)
throws DatabaseException, MissingParamException, RecordNotFoundException {

MissingParam.checkMissing(conn, "conn");
MissingParam.checkPositive(datasetId, "datasetId");

try (PreparedStatement stmt = conn
.prepareStatement(DATASET_EXPORTED_STATEMENT)) {
stmt.setLong(1, datasetId);
stmt.execute();
PreparedStatement setExportedStmt = null;
PreparedStatement setLastNrtStmt = null;

try {
conn.setAutoCommit(false);

setExportedStmt = conn.prepareStatement(DATASET_EXPORTED_STATEMENT);
setExportedStmt.setLong(1, datasetId);
setExportedStmt.execute();

if (recordNrt) {
DataSet dataset = getDataSet(conn, datasetId);
if (dataset.isNrt()) {
setLastNrtStmt = conn.prepareStatement(LAST_NRT_STATEMENT);
setLastNrtStmt.setLong(1,
DateTimeUtils.dateToLong(LocalDateTime.now()));
setLastNrtStmt.setLong(2, dataset.getInstrumentId());

setLastNrtStmt.execute();
}
}

conn.commit();
} catch (SQLException e) {
throw new DatabaseException("Error setting dataset export status", e);
} finally {
DatabaseUtils.closeStatements(setExportedStmt, setLastNrtStmt);

try {
conn.setAutoCommit(true);
} catch (SQLException e) {
// Close the connection so it can't be reused
DatabaseUtils.closeConnection(conn);
throw new DatabaseException("Error setting dataset export status", e);
}
}
}
}
25 changes: 20 additions & 5 deletions WebApp/src/uk/ac/exeter/QuinCe/data/Instrument/Instrument.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package uk.ac.exeter.QuinCe.data.Instrument;

import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -136,6 +137,12 @@ public class Instrument {
*/
private boolean nrt = false;

/**
* The time at which the most recent automatic export of an NRT dataset was
* performed for this Instrument.
*/
private LocalDateTime lastNrtExport = null;

/**
* The configuration for the behaviour of diagnostic sensors QC.
*/
Expand Down Expand Up @@ -169,8 +176,8 @@ public Instrument(User owner, long databaseId, String name,
InstrumentFileSet fileDefinitions, List<Variable> variables,
Map<Variable, Properties> variableProperties,
SensorAssignments sensorAssignments, String platformName,
String platformCode, boolean nrt, String propertiesJson)
throws SensorGroupsException {
String platformCode, boolean nrt, LocalDateTime lastNrtExport,
String propertiesJson) throws SensorGroupsException {

this.owner = owner;
this.id = databaseId;
Expand All @@ -182,6 +189,7 @@ public Instrument(User owner, long databaseId, String name,
this.platformName = platformName;
this.platformCode = platformCode;
this.nrt = nrt;
this.lastNrtExport = lastNrtExport;
parsePropertiesJson(propertiesJson);
}

Expand Down Expand Up @@ -210,8 +218,8 @@ public Instrument(User owner, long databaseId, String name,
public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
List<Variable> variables, Map<Variable, Properties> variableProperties,
SensorAssignments sensorAssignments, String platformName,
String platformCode, boolean nrt, String propertiesJson)
throws SensorGroupsException {
String platformCode, boolean nrt, LocalDateTime lastNrtExport,
String propertiesJson) throws SensorGroupsException {

this.owner = owner;
this.name = name;
Expand All @@ -222,6 +230,7 @@ public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
this.platformName = platformName;
this.platformCode = platformCode;
this.nrt = nrt;
this.lastNrtExport = lastNrtExport;
parsePropertiesJson(propertiesJson);
}

Expand All @@ -247,7 +256,8 @@ public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
List<Variable> variables, Map<Variable, Properties> variableProperties,
SensorAssignments sensorAssignments, SensorGroups sensorGroups,
String platformName, String platformCode, boolean nrt) {
String platformName, String platformCode, boolean nrt,
LocalDateTime lastNrtExport) {

this.owner = owner;
this.name = name;
Expand All @@ -259,6 +269,7 @@ public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
this.platformName = platformName;
this.platformCode = platformCode;
this.nrt = nrt;
this.lastNrtExport = lastNrtExport;
this.properties = new Properties();
}

Expand Down Expand Up @@ -1061,4 +1072,8 @@ public List<Variable> getVariables(Collection<Long> ids)

return result;
}

public LocalDateTime getLastNrtExport() {
return lastNrtExport;
}
}
Loading

0 comments on commit 2fb1b81

Please sign in to comment.