Skip to content

Commit

Permalink
Convert MATS to use surfrad4 (#1197)
Browse files Browse the repository at this point in the history 
This PR moves MATS compatibility over to the faster surfrad4 database
and adds dieoff plots.

Note: Histogram results are different in this version of the code, but I
have determined that the new plots are the correct ones.
  • Loading branch information
@JeffHamiltonNOAA
JeffHamiltonNOAA committed Jun 24, 2024
2 parents ba7b122 + 5d2bcda commit b8fde47
Show file tree
Hide file tree
Showing 33 changed files with 1,466 additions and 318 deletions.
2 changes: 1 addition & 1 deletion MATScommon
Submodule MATScommon updated 2 files
+1 −0 meteor_packages/mats-common/imports/startup/both/mats-curve-params.js
+3 −2 meteor_packages/mats-common/public/MATSReleaseNotes.html
2 changes: 1 addition & 1 deletion METexpress
Submodule METexpress updated 1 files
+1 −1 MATScommon
2 changes: 1 addition & 1 deletion apps/ceil-vis15/server/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,7 +457,7 @@ const doCurveParams = function () {
.split(",")
.map(Function.prototype.call, String.prototype.trim)
.map(function (fhr) {
return Number(fhr.replace(/'|\[|\]/g, "")) / 60;
return (Number(fhr.replace(/'|\[|\]/g, "")) / 60).toString();
});

const thresholds = rows[i].trsh;
Expand Down
52 changes: 25 additions & 27 deletions apps/surfrad/server/dataFunctions/data_contour.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ dataContour = function (plotParams, plotFunction) {
{ name: "variable" },
{ optionsMap: 1 }
).optionsMap;
const variable = variableOptionsMap[variableStr];

let scaleClause = "";
const scaleStr = curve.scale;
Expand Down Expand Up @@ -105,13 +104,9 @@ dataContour = function (plotParams, plotFunction) {
{ name: "statistic" },
{ optionsMap: 1 }
).optionsMap;
const statisticClause =
`sum(${variable[0]}) as square_diff_sum, count(${variable[1]}) as N_sum, sum(${variable[2]}) as obs_model_diff_sum, sum(${variable[3]}) as model_sum, sum(${variable[4]}) as obs_sum, sum(${variable[5]}) as abs_sum, ` +
`group_concat(m0.secs, ';', ${variable[0]}, ';', 1, ';', ${variable[2]}, ';', ${variable[3]}, ';', ${variable[4]}, ';', ${variable[5]} order by m0.secs) as sub_data, count(${variable[0]}) as N0`;

let dateString = "";
let dateClause = "";
let matchClause = "";
if (
(xAxisParam === "Init Date" || yAxisParam === "Init Date") &&
xAxisParam !== "Valid Date" &&
Expand All @@ -121,8 +116,7 @@ dataContour = function (plotParams, plotFunction) {
} else {
dateString = "m0.secs";
}
dateClause = `and o.secs >= ${fromSecs} and o.secs <= ${toSecs} and ${dateString} >= ${fromSecs} and ${dateString} <= ${toSecs}`;
matchClause = "and m0.id = o.id and m0.secs = o.secs";
dateClause = `and ${dateString} >= ${fromSecs} and ${dateString} <= ${toSecs}`;

const regionStr = curve.region;
const region = Object.keys(
Expand All @@ -131,18 +125,36 @@ dataContour = function (plotParams, plotFunction) {
(key) =>
matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr
);
let regionClause;

let queryTableClause;
let NAggregate;
let NClause;
let variable;
if (region === "all_stat") {
regionClause = "";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_site_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else if (region === "all_surf") {
regionClause = "and m0.id in(1,2,3,4,5,6,7) ";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_surfrad_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else if (region === "all_sol") {
regionClause = "and m0.id in(8,9,10,11,12,13,14) ";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_solrad_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else {
regionClause = `and m0.id in(${region}) `;
variable = variableOptionsMap[variableStr]["Select stations"];
queryTableClause = `from ${model}_site_${region} as m0`;
NAggregate = "count";
NClause = "1";
}

const queryTableClause = `from surfrad as o, ${model} as m0`;
const statisticClause =
`sum(${variable[0]}) as square_diff_sum, ${NAggregate}(${variable[1]}) as N_sum, sum(${variable[2]}) as obs_model_diff_sum, sum(${variable[3]}) as model_sum, sum(${variable[4]}) as obs_sum, sum(${variable[5]}) as abs_sum, ` +
`group_concat(m0.secs, ';', ${variable[0]}, ';', ${NClause}, ';', ${variable[2]}, ';', ${variable[3]}, ';', ${variable[4]}, ';', ${variable[5]} order by m0.secs) as sub_data, count(${variable[0]}) as N0`;

// For contours, this functions as the colorbar label.
const { statVarUnitMap } = matsCollections.variable.findOne(
Expand All @@ -168,12 +180,10 @@ dataContour = function (plotParams, plotFunction) {
"{{statisticClause}} " +
"{{queryTableClause}} " +
"where 1=1 " +
"{{matchClause}} " +
"{{dateClause}} " +
"{{validTimeClause}} " +
"{{forecastLengthClause}} " +
"{{scaleClause}} " +
"{{regionClause}} " +
"group by xVal,yVal " +
"order by xVal,yVal" +
";";
Expand All @@ -185,22 +195,10 @@ dataContour = function (plotParams, plotFunction) {
statement = statement.replace("{{validTimeClause}}", validTimeClause);
statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause);
statement = statement.replace("{{scaleClause}}", scaleClause);
statement = statement.replace("{{regionClause}}", regionClause);
statement = statement.replace("{{matchClause}}", matchClause);
statement = statement.replace("{{dateClause}}", dateClause);
statement = statement.split("{{dateString}}").join(dateString);
dataRequests[label] = statement;

if (
model !== "HRRR" &&
variableStr !== "dswrf" &&
statisticSelect !== "Obs average"
) {
throw new Error(
`INFO: The statistic/variable combination [${statisticSelect} and ${variableStr}] is only available for the HRRR data-source.`
);
}

// send the query statement to the query function
queryResult = matsDataQueryUtils.queryDBContour(
sumPool, // eslint-disable-line no-undef
Expand Down
52 changes: 25 additions & 27 deletions apps/surfrad/server/dataFunctions/data_contour_diff.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ dataContourDiff = function (plotParams, plotFunction) {
{ name: "variable" },
{ optionsMap: 1 }
).optionsMap;
const variable = variableOptionsMap[variableStr];

let scaleClause = "";
const scaleStr = curve.scale;
Expand Down Expand Up @@ -114,13 +113,9 @@ dataContourDiff = function (plotParams, plotFunction) {
{ name: "statistic" },
{ optionsMap: 1 }
).optionsMap;
const statisticClause =
`sum(${variable[0]}) as square_diff_sum, count(${variable[1]}) as N_sum, sum(${variable[2]}) as obs_model_diff_sum, sum(${variable[3]}) as model_sum, sum(${variable[4]}) as obs_sum, sum(${variable[5]}) as abs_sum, ` +
`group_concat(m0.secs, ';', ${variable[0]}, ';', 1, ';', ${variable[2]}, ';', ${variable[3]}, ';', ${variable[4]}, ';', ${variable[5]} order by m0.secs) as sub_data, count(${variable[0]}) as N0`;

let dateString = "";
let dateClause = "";
let matchClause = "";
if (
(xAxisParam === "Init Date" || yAxisParam === "Init Date") &&
xAxisParam !== "Valid Date" &&
Expand All @@ -130,8 +125,7 @@ dataContourDiff = function (plotParams, plotFunction) {
} else {
dateString = "m0.secs";
}
dateClause = `and o.secs >= ${fromSecs} and o.secs <= ${toSecs} and ${dateString} >= ${fromSecs} and ${dateString} <= ${toSecs}`;
matchClause = "and m0.id = o.id and m0.secs = o.secs";
dateClause = `and ${dateString} >= ${fromSecs} and ${dateString} <= ${toSecs}`;

const regionStr = curve.region;
const region = Object.keys(
Expand All @@ -140,18 +134,36 @@ dataContourDiff = function (plotParams, plotFunction) {
(key) =>
matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr
);
let regionClause;

let queryTableClause;
let NAggregate;
let NClause;
let variable;
if (region === "all_stat") {
regionClause = "";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_site_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else if (region === "all_surf") {
regionClause = "and m0.id in(1,2,3,4,5,6,7) ";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_surfrad_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else if (region === "all_sol") {
regionClause = "and m0.id in(8,9,10,11,12,13,14) ";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_solrad_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else {
regionClause = `and m0.id in(${region}) `;
variable = variableOptionsMap[variableStr]["Select stations"];
queryTableClause = `from ${model}_site_${region} as m0`;
NAggregate = "count";
NClause = "1";
}

const queryTableClause = `from surfrad as o, ${model} as m0`;
const statisticClause =
`sum(${variable[0]}) as square_diff_sum, ${NAggregate}(${variable[1]}) as N_sum, sum(${variable[2]}) as obs_model_diff_sum, sum(${variable[3]}) as model_sum, sum(${variable[4]}) as obs_sum, sum(${variable[5]}) as abs_sum, ` +
`group_concat(m0.secs, ';', ${variable[0]}, ';', ${NClause}, ';', ${variable[2]}, ';', ${variable[3]}, ';', ${variable[4]}, ';', ${variable[5]} order by m0.secs) as sub_data, count(${variable[0]}) as N0`;

// For contours, this functions as the colorbar label.
const { statVarUnitMap } = matsCollections.variable.findOne(
Expand Down Expand Up @@ -179,12 +191,10 @@ dataContourDiff = function (plotParams, plotFunction) {
"{{statisticClause}} " +
"{{queryTableClause}} " +
"where 1=1 " +
"{{matchClause}} " +
"{{dateClause}} " +
"{{validTimeClause}} " +
"{{forecastLengthClause}} " +
"{{scaleClause}} " +
"{{regionClause}} " +
"group by xVal,yVal " +
"order by xVal,yVal" +
";";
Expand All @@ -196,22 +206,10 @@ dataContourDiff = function (plotParams, plotFunction) {
statement = statement.replace("{{validTimeClause}}", validTimeClause);
statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause);
statement = statement.replace("{{scaleClause}}", scaleClause);
statement = statement.replace("{{regionClause}}", regionClause);
statement = statement.replace("{{matchClause}}", matchClause);
statement = statement.replace("{{dateClause}}", dateClause);
statement = statement.split("{{dateString}}").join(dateString);
dataRequests[label] = statement;

if (
model !== "HRRR" &&
variableStr !== "dswrf" &&
statisticSelect !== "Obs average"
) {
throw new Error(
`INFO: The statistic/variable combination [${statisticSelect} and ${variableStr}] is only available for the HRRR data-source.`
);
}

// send the query statement to the query function
queryResult = matsDataQueryUtils.queryDBContour(
sumPool, // eslint-disable-line no-undef
Expand Down
51 changes: 25 additions & 26 deletions apps/surfrad/server/dataFunctions/data_dailymodelcycle.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ dataDailyModelCycle = function (plotParams, plotFunction) {
{ name: "variable" },
{ optionsMap: 1 }
).optionsMap;
const variable = variableOptionsMap[variableStr];

const scaleStr = curve.scale;
const scale = Object.keys(
Expand All @@ -87,12 +86,8 @@ dataDailyModelCycle = function (plotParams, plotFunction) {
{ name: "statistic" },
{ optionsMap: 1 }
).optionsMap;
const statisticClause =
`sum(${variable[0]}) as square_diff_sum, count(${variable[1]}) as N_sum, sum(${variable[2]}) as obs_model_diff_sum, sum(${variable[3]}) as model_sum, sum(${variable[4]}) as obs_sum, sum(${variable[5]}) as abs_sum, ` +
`group_concat(m0.secs, ';', ${variable[0]}, ';', 1, ';', ${variable[2]}, ';', ${variable[3]}, ';', ${variable[4]}, ';', ${variable[5]} order by m0.secs) as sub_data, count(${variable[0]}) as N0`;

const dateClause = `and o.secs >= ${fromSecs} and o.secs <= ${toSecs} and m0.secs >= ${fromSecs} and m0.secs <= ${toSecs}`;
const matchClause = "and m0.id = o.id and m0.secs = o.secs";
const dateClause = `and m0.secs >= ${fromSecs} and m0.secs <= ${toSecs}`;

const regionStr = curve.region;
const region = Object.keys(
Expand All @@ -101,18 +96,36 @@ dataDailyModelCycle = function (plotParams, plotFunction) {
(key) =>
matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr
);
let regionClause;

let queryTableClause;
let NAggregate;
let NClause;
let variable;
if (region === "all_stat") {
regionClause = "";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_site_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else if (region === "all_surf") {
regionClause = "and m0.id in(1,2,3,4,5,6,7) ";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_surfrad_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else if (region === "all_sol") {
regionClause = "and m0.id in(8,9,10,11,12,13,14) ";
variable = variableOptionsMap[variableStr]["Predefined region"];
queryTableClause = `from ${model}_all_solrad_sums as m0`;
NAggregate = "sum";
[, NClause] = variable;
} else {
regionClause = `and m0.id in(${region}) `;
variable = variableOptionsMap[variableStr]["Select stations"];
queryTableClause = `from ${model}_site_${region} as m0`;
NAggregate = "count";
NClause = "1";
}

const queryTableClause = `from surfrad as o, ${model} as m0`;
const statisticClause =
`sum(${variable[0]}) as square_diff_sum, ${NAggregate}(${variable[1]}) as N_sum, sum(${variable[2]}) as obs_model_diff_sum, sum(${variable[3]}) as model_sum, sum(${variable[4]}) as obs_sum, sum(${variable[5]}) as abs_sum, ` +
`group_concat(m0.secs, ';', ${variable[0]}, ';', ${NClause}, ';', ${variable[2]}, ';', ${variable[3]}, ';', ${variable[4]}, ';', ${variable[5]} order by m0.secs) as sub_data, count(${variable[0]}) as N0`;

// axisKey is used to determine which axis a curve should use.
// This axisKeySet object is used like a set and if a curve has the same
Expand Down Expand Up @@ -141,12 +154,10 @@ dataDailyModelCycle = function (plotParams, plotFunction) {
"{{statisticClause}} " +
"{{queryTableClause}} " +
"where 1=1 " +
"{{matchClause}} " +
"{{dateClause}} " +
"{{utcCycleStartClause}} " +
"{{forecastLengthClause}} " +
"{{scaleClause}} " +
"{{regionClause}} " +
"group by avtime " +
"order by avtime" +
";";
Expand All @@ -156,21 +167,9 @@ dataDailyModelCycle = function (plotParams, plotFunction) {
statement = statement.replace("{{utcCycleStartClause}}", utcCycleStartClause);
statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause);
statement = statement.replace("{{scaleClause}}", scaleClause);
statement = statement.replace("{{regionClause}}", regionClause);
statement = statement.replace("{{matchClause}}", matchClause);
statement = statement.replace("{{dateClause}}", dateClause);
dataRequests[label] = statement;

if (
model !== "HRRR" &&
variableStr !== "dswrf" &&
statisticSelect !== "Obs average"
) {
throw new Error(
`INFO: The statistic/variable combination [${statisticSelect} and ${variableStr}] is only available for the HRRR data-source.`
);
}

// send the query statement to the query function
queryResult = matsDataQueryUtils.queryDBSpecialtyCurve(
sumPool, // eslint-disable-line no-undef
Expand Down
Loading

0 comments on commit b8fde47

Please sign in to comment.