From 4e52500dd608a093571939e1436fa0a87452ae54 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Thu, 9 Nov 2023 18:11:30 -0700 Subject: [PATCH] fully linted scorecard --- apps/ensemble/.eslintrc.json | 4 +- apps/ensemble/client/main.js | 1 + .../server/dataFunctions/data_contour.js | 299 +++++++++-------- .../server/dataFunctions/data_contour_diff.js | 304 ++++++++++-------- .../server/dataFunctions/data_dieoff.js | 174 +++++----- .../dataFunctions/data_gridscale_prob.js | 131 ++++---- .../server/dataFunctions/data_histogram.js | 157 +++++---- .../server/dataFunctions/data_reliability.js | 131 ++++---- .../server/dataFunctions/data_series.js | 185 ++++++----- .../server/dataFunctions/data_threshold.js | 151 +++++---- .../server/dataFunctions/data_validtime.js | 144 +++++---- apps/ensemble/server/main.js | 139 ++++---- 12 files changed, 1012 insertions(+), 808 deletions(-) diff --git a/apps/ensemble/.eslintrc.json b/apps/ensemble/.eslintrc.json index a33ac158ff..4c1bfbc4b2 100644 --- a/apps/ensemble/.eslintrc.json +++ b/apps/ensemble/.eslintrc.json @@ -28,13 +28,11 @@ "space-before-function-paren": "off", // for Meteor API's that rely on `this` context, e.g. Template.onCreated and publications "func-names": "off", - "prefer-arrow-callback": "off", + "prefer-arrow-callback": "off" // Vx Team modifications - Warn on rules that would require refactoring to implement. // We want to be able to turn these back into "error"'s at some point. However, for // our first pass, we'll only consider the checks that ESLint can auto-fix as errors. // https://eslint.org/docs/latest/use/configure/rules#rule-severities - "no-undef": "warn", - "no-unused-vars": "warn" } } diff --git a/apps/ensemble/client/main.js b/apps/ensemble/client/main.js index a87407a1f4..ecd922b6a2 100644 --- a/apps/ensemble/client/main.js +++ b/apps/ensemble/client/main.js @@ -2,6 +2,7 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ +// eslint-disable-next-line no-unused-vars import { matsTypes, matsCollections, methods } from "meteor/randyp:mats-common"; import "@fortawesome/fontawesome-free"; import "@fortawesome/fontawesome-free/css/all.css"; diff --git a/apps/ensemble/server/dataFunctions/data_contour.js b/apps/ensemble/server/dataFunctions/data_contour.js index 06573192c3..9b85835a27 100644 --- a/apps/ensemble/server/dataFunctions/data_contour.js +++ b/apps/ensemble/server/dataFunctions/data_contour.js @@ -12,6 +12,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataContour = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -22,90 +23,91 @@ dataContour = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; - const totalProcessingStart = moment(); + + const curves = JSON.parse(JSON.stringify(plotParams.curves)); + if (curves.length > 1) { + throw new Error("INFO: There must only be one added curve."); + } + + const axisMap = Object.create(null); + + let statement = ""; + let error = ""; + const dataset = []; + const dateRange = matsDataUtils.getDateRange(plotParams.dates); const fromSecs = dateRange.fromSeconds; const toSecs = dateRange.toSeconds; + const xAxisParam = plotParams["x-axis-parameter"]; const yAxisParam = plotParams["y-axis-parameter"]; const xValClause = matsCollections.PlotParams.findOne({ name: "x-axis-parameter" }) .optionsMap[xAxisParam]; const yValClause = matsCollections.PlotParams.findOne({ name: "y-axis-parameter" }) .optionsMap[yAxisParam]; - let error = ""; - const curves = JSON.parse(JSON.stringify(plotParams.curves)); - if (curves.length > 1) { - throw new Error("INFO: There must only be one added curve."); - } - const dataset = []; - const axisMap = Object.create(null); - // initialize variables specific to the curve + // initialize variables specific to this curve const curve = curves[0]; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }).optionsMap[ variable ]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const statisticSelect = curve.statistic; - const statisticOptionsMap = matsCollections.statistic.findOne( - { name: "statistic" }, - { optionsMap: 1 } - ).optionsMap[appParams.plotType]; - const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; - const { members } = curve; - const memberClause = `and m0.mem = ${members}`; - const neighborhoodSize = curve["neighborhood-size"]; - const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; - let kernelClause = ""; - let probBinClause = ""; - let radiusClause = ""; - if (tableStatPrefix === "count") { - const { kernel } = curve; - kernelClause = `and m0.kernel = ${kernel}`; - const probBins = - curve["probability-bins"] === undefined ? [] : curve["probability-bins"]; - if (probBins.length !== 0 && probBins !== matsTypes.InputTypes.unused) { - probBinClause = `and m0.prob IN(${probBins})`; - } else { - throw new Error("INFO: You need to select at least one probability bin."); - } - } else { - const { radius } = curve; - radiusClause = `and m0.radius = ${radius}`; - } + let thresholdClause = ""; - let validTimeClause = ""; - let forecastLengthClause = ""; - let dateString = ""; - let dateClause = ""; if (xAxisParam !== "Threshold" && yAxisParam !== "Threshold") { const { threshold } = curve; + if (threshold === undefined) { + throw new Error( + `INFO: ${label}'s threshold is undefined. Please assign it a value.` + ); + } thresholdClause = `and m0.trsh = ${threshold}`; } + + const { members } = curve; + const memberClause = `and m0.mem = ${members}`; + + const neighborhoodSize = curve["neighborhood-size"]; + const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + let validTimeClause = ""; if (xAxisParam !== "Valid UTC hour" && yAxisParam !== "Valid UTC hour") { const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; - if (validTimes.length > 0 && validTimes !== matsTypes.InputTypes.unused) { + if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { validTimeClause = `and m0.time%(24*3600)/3600 IN(${validTimes})`; } } + + let forecastLengthClause = ""; if (xAxisParam !== "Fcst lead time" && yAxisParam !== "Fcst lead time") { const forecastLength = curve["forecast-length"]; + if (forecastLength === undefined) { + throw new Error( + `INFO: ${label}'s forecast lead time is undefined. Please assign it a value.` + ); + } forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; } + + const statisticSelect = curve.statistic; + const statisticOptionsMap = matsCollections.statistic.findOne( + { name: "statistic" }, + { optionsMap: 1 } + ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; + const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; + + let dateString = ""; + let dateClause = ""; if ( (xAxisParam === "Init Date" || yAxisParam === "Init Date") && xAxisParam !== "Valid Date" && @@ -116,93 +118,130 @@ dataContour = function (plotParams, plotFunction) { dateString = "m0.time"; } dateClause = `and ${dateString} >= ${fromSecs} and ${dateString} <= ${toSecs}`; - const [statisticClause] = statisticOptionsMap[statisticSelect]; + + let kernelClause = ""; + let probBinClause = ""; + let radiusClause = ""; + if (tableStatPrefix === "count") { + const { kernel } = curve; + kernelClause = `and m0.kernel = ${kernel}`; + const probBins = + curve["probability-bins"] === undefined ? [] : curve["probability-bins"]; + if (probBins.length !== 0 && probBins !== matsTypes.InputTypes.unused) { + probBinClause = `and m0.prob IN(${probBins})`; + } else { + throw new Error("INFO: You need to select at least one probability bin."); + } + } else { + const { radius } = curve; + radiusClause = `and m0.radius = ${radius}`; + } + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // For contours, this functions as the colorbar label. const statType = statisticOptionsMap[statisticSelect][0]; [, , , curve.unitKey] = statisticOptionsMap[statisticSelect]; let d; - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "{{xValClause}} " + - "{{yValClause}} " + - "count(distinct {{dateString}}) as N_times, " + - "min({{dateString}}) as min_secs, " + - "max({{dateString}}) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by xVal,yVal " + - "order by xVal,yVal" + - ";"; - - statement = statement.replace("{{xValClause}}", xValClause); - statement = statement.replace("{{yValClause}}", yValClause); - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - statement = statement.split("{{dateString}}").join(dateString); - dataRequests[label] = statement; - - let queryResult; - const startMoment = moment(); - let finishMoment; - try { - // send the query statement to the query function - queryResult = matsDataQueryUtils.queryDBContour( - sumPool, - statement, - appParams, - statisticSelect - ); - finishMoment = moment(); - dataRequests[`data retrieval (query) time - ${label}`] = { - begin: startMoment.format(), - finish: finishMoment.format(), - duration: `${moment - .duration(finishMoment.diff(startMoment)) - .asSeconds()} seconds`, - recordCount: queryResult.data.xTextOutput.length, - }; - // get the data back from the query - d = queryResult.data; - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; - throw new Error(e.message); - } - if (queryResult.error !== undefined && queryResult.error !== "") { - if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { - // this is NOT an error just a no data condition - dataFoundForCurve = false; - } else { - // this is an error returned by the mysql database - error += `Error from verification query:
${queryResult.error}
query:
${statement}
`; - throw new Error(error); + if (!diffFrom) { + let queryResult; + const startMoment = moment(); + let finishMoment; + try { + statement = + "{{xValClause}} " + + "{{yValClause}} " + + "count(distinct {{dateString}}) as N_times, " + + "min({{dateString}}) as min_secs, " + + "max({{dateString}}) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by xVal,yVal " + + "order by xVal,yVal" + + ";"; + + statement = statement.replace("{{xValClause}}", xValClause); + statement = statement.replace("{{yValClause}}", yValClause); + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + statement = statement.split("{{dateString}}").join(dateString); + dataRequests[label] = statement; + + // send the query statement to the query function + queryResult = matsDataQueryUtils.queryDBContour( + sumPool, // eslint-disable-line no-undef + statement, + appParams, + statisticSelect + ); + + finishMoment = moment(); + dataRequests[label] = statement; + dataRequests[`data retrieval (query) time - ${label}`] = { + begin: startMoment.format(), + finish: finishMoment.format(), + duration: `${moment + .duration(finishMoment.diff(startMoment)) + .asSeconds()} seconds`, + recordCount: queryResult.data.xTextOutput.length, + }; + // get the data back from the query + d = queryResult.data; + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; + throw new Error(e.message); + } + + if (queryResult.error !== undefined && queryResult.error !== "") { + if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { + // this is NOT an error just a no data condition + dataFoundForCurve = false; + } else { + // this is an error returned by the mysql database + error += `Error from verification query:
${queryResult.error}
query:
${statement}
`; + throw new Error(error); + } } - } - if (!dataFoundForCurve) { - // we found no data for any curves so don't bother proceeding - throw new Error("INFO: No valid data for any curves."); + if (!dataFoundForCurve) { + // we found no data for any curves so don't bother proceeding + throw new Error("INFO: No valid data for any curves."); + } + } else { + // this is a difference curve -- not supported for contours + throw new Error( + "INFO: Difference curves are not supported for contours, as there is only one curve." + ); } // set curve annotation to be the curve mean -- may be recalculated later diff --git a/apps/ensemble/server/dataFunctions/data_contour_diff.js b/apps/ensemble/server/dataFunctions/data_contour_diff.js index 1e7a2a04f0..40b0dc0b33 100644 --- a/apps/ensemble/server/dataFunctions/data_contour_diff.js +++ b/apps/ensemble/server/dataFunctions/data_contour_diff.js @@ -14,6 +14,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataContourDiff = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -24,95 +25,96 @@ dataContourDiff = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries - let dataFoundForCurve = true; let dataNotFoundForAnyCurve = false; + + let curves = JSON.parse(JSON.stringify(plotParams.curves)); + const curvesLength = curves.length; + if (curvesLength !== 2) { + throw new Error("INFO: There must be two added curves."); + } + + const axisMap = Object.create(null); const showSignificance = plotParams.significance !== "none"; - const totalProcessingStart = moment(); + + let statType; + let statisticSelect; + + let statement = ""; + let error = ""; + let dataset = []; + const dateRange = matsDataUtils.getDateRange(plotParams.dates); const fromSecs = dateRange.fromSeconds; const toSecs = dateRange.toSeconds; + const xAxisParam = plotParams["x-axis-parameter"]; const yAxisParam = plotParams["y-axis-parameter"]; const xValClause = matsCollections.PlotParams.findOne({ name: "x-axis-parameter" }) .optionsMap[xAxisParam]; const yValClause = matsCollections.PlotParams.findOne({ name: "y-axis-parameter" }) .optionsMap[yAxisParam]; - let error = ""; - let curves = JSON.parse(JSON.stringify(plotParams.curves)); - const curvesLength = curves.length; - if (curvesLength !== 2) { - throw new Error("INFO: There must be two added curves."); - } - let statType; - let statisticSelect; - let dataset = []; - const axisMap = Object.create(null); for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - statisticSelect = curve.statistic; - const statisticOptionsMap = matsCollections.statistic.findOne( - { name: "statistic" }, - { optionsMap: 1 } - ).optionsMap[appParams.plotType]; - const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; - const { members } = curve; - const memberClause = `and m0.mem = ${members}`; - const neighborhoodSize = curve["neighborhood-size"]; - const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; - let kernelClause = ""; - let probBinClause = ""; - let radiusClause = ""; - if (tableStatPrefix === "count") { - const { kernel } = curve; - kernelClause = `and m0.kernel = ${kernel}`; - const probBins = - curve["probability-bins"] === undefined ? [] : curve["probability-bins"]; - if (probBins.length !== 0 && probBins !== matsTypes.InputTypes.unused) { - probBinClause = `and m0.prob IN(${probBins})`; - } else { - throw new Error("INFO: You need to select at least one probability bin."); - } - } else { - const { radius } = curve; - radiusClause = `and m0.radius = ${radius}`; - } + let thresholdClause = ""; - let validTimeClause = ""; - let forecastLengthClause = ""; - let dateString = ""; - let dateClause = ""; if (xAxisParam !== "Threshold" && yAxisParam !== "Threshold") { const { threshold } = curve; + if (threshold === undefined) { + throw new Error( + `INFO: ${label}'s threshold is undefined. Please assign it a value.` + ); + } thresholdClause = `and m0.trsh = ${threshold}`; } + + const { members } = curve; + const memberClause = `and m0.mem = ${members}`; + + const neighborhoodSize = curve["neighborhood-size"]; + const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + let validTimeClause = ""; if (xAxisParam !== "Valid UTC hour" && yAxisParam !== "Valid UTC hour") { const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; - if (validTimes.length > 0 && validTimes !== matsTypes.InputTypes.unused) { + if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { validTimeClause = `and m0.time%(24*3600)/3600 IN(${validTimes})`; } } + + let forecastLengthClause = ""; if (xAxisParam !== "Fcst lead time" && yAxisParam !== "Fcst lead time") { const forecastLength = curve["forecast-length"]; + if (forecastLength === undefined) { + throw new Error( + `INFO: ${label}'s forecast lead time is undefined. Please assign it a value.` + ); + } forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; } + + statisticSelect = curve.statistic; + const statisticOptionsMap = matsCollections.statistic.findOne( + { name: "statistic" }, + { optionsMap: 1 } + ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; + const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; + + let dateString = ""; + let dateClause = ""; if ( (xAxisParam === "Init Date" || yAxisParam === "Init Date") && xAxisParam !== "Valid Date" && @@ -123,89 +125,123 @@ dataContourDiff = function (plotParams, plotFunction) { dateString = "m0.time"; } dateClause = `and ${dateString} >= ${fromSecs} and ${dateString} <= ${toSecs}`; - const [statisticClause] = statisticOptionsMap[statisticSelect]; + + let kernelClause = ""; + let probBinClause = ""; + let radiusClause = ""; + if (tableStatPrefix === "count") { + const { kernel } = curve; + kernelClause = `and m0.kernel = ${kernel}`; + const probBins = + curve["probability-bins"] === undefined ? [] : curve["probability-bins"]; + if (probBins.length !== 0 && probBins !== matsTypes.InputTypes.unused) { + probBinClause = `and m0.prob IN(${probBins})`; + } else { + throw new Error("INFO: You need to select at least one probability bin."); + } + } else { + const { radius } = curve; + radiusClause = `and m0.radius = ${radius}`; + } + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // For contours, this functions as the colorbar label. [statType] = statisticOptionsMap[statisticSelect]; [, , , curve.unitKey] = statisticOptionsMap[statisticSelect]; let d; - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "{{xValClause}} " + - "{{yValClause}} " + - "count(distinct {{dateString}}) as N_times, " + - "min({{dateString}}) as min_secs, " + - "max({{dateString}}) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by xVal,yVal " + - "order by xVal,yVal" + - ";"; + if (!diffFrom) { + let queryResult; + const startMoment = moment(); + let finishMoment; + try { + statement = + "{{xValClause}} " + + "{{yValClause}} " + + "count(distinct {{dateString}}) as N_times, " + + "min({{dateString}}) as min_secs, " + + "max({{dateString}}) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by xVal,yVal " + + "order by xVal,yVal" + + ";"; - statement = statement.replace("{{xValClause}}", xValClause); - statement = statement.replace("{{yValClause}}", yValClause); - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - statement = statement.split("{{dateString}}").join(dateString); - dataRequests[label] = statement; + statement = statement.replace("{{xValClause}}", xValClause); + statement = statement.replace("{{yValClause}}", yValClause); + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + statement = statement.split("{{dateString}}").join(dateString); + dataRequests[label] = statement; - let queryResult; - const startMoment = moment(); - let finishMoment; - try { - // send the query statement to the query function - queryResult = matsDataQueryUtils.queryDBContour( - sumPool, - statement, - appParams, - statisticSelect - ); - finishMoment = moment(); - dataRequests[`data retrieval (query) time - ${label}`] = { - begin: startMoment.format(), - finish: finishMoment.format(), - duration: `${moment - .duration(finishMoment.diff(startMoment)) - .asSeconds()} seconds`, - recordCount: queryResult.data.xTextOutput.length, - }; - // get the data back from the query - d = queryResult.data; - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; - throw new Error(e.message); - } - if (queryResult.error !== undefined && queryResult.error !== "") { - if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { - // this is NOT an error just a no data condition - dataFoundForCurve = false; - } else { - // this is an error returned by the mysql database - error += `Error from verification query:
${queryResult.error}
query:
${statement}
`; - throw new Error(error); + // send the query statement to the query function + queryResult = matsDataQueryUtils.queryDBContour( + sumPool, // eslint-disable-line no-undef + statement, + appParams, + statisticSelect + ); + + finishMoment = moment(); + dataRequests[label] = statement; + dataRequests[`data retrieval (query) time - ${label}`] = { + begin: startMoment.format(), + finish: finishMoment.format(), + duration: `${moment + .duration(finishMoment.diff(startMoment)) + .asSeconds()} seconds`, + recordCount: queryResult.data.xTextOutput.length, + }; + // get the data back from the query + d = queryResult.data; + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; + throw new Error(e.message); + } + + if (queryResult.error !== undefined && queryResult.error !== "") { + if (queryResult.error !== matsTypes.Messages.NO_DATA_FOUND) { + // this is an error returned by the mysql database + error += `Error from verification query:
${queryResult.error}
query:
${statement}
`; + throw new Error(error); + } + dataNotFoundForAnyCurve = true; } - dataNotFoundForAnyCurve = true; + } else { + // this is a difference curve -- not supported for contours + throw new Error( + "INFO: Difference curves are not supported for contours, as there is only one curve." + ); } // set curve annotation to be the curve mean -- may be recalculated later @@ -259,13 +295,9 @@ dataContourDiff = function (plotParams, plotFunction) { statType === "ctc", statType === "scalar" ); - - // make a copy of the plotParams that we can modify - const diffPlotParams = JSON.parse(JSON.stringify(plotParams)); - diffPlotParams.curves = matsDataUtils.getDiffContourCurveParams( - diffPlotParams.curves - ); - curves = diffPlotParams.curves; + const newPlotParams = plotParams; + newPlotParams.curves = matsDataUtils.getDiffContourCurveParams(plotParams.curves); + curves = newPlotParams.curves; dataset[0].name = matsPlotUtils.getCurveText( matsTypes.PlotTypes.contourDiff, curves[0] @@ -281,7 +313,7 @@ dataContourDiff = function (plotParams, plotFunction) { const result = matsDataProcessUtils.processDataContour( dataset, curveInfoParams, - diffPlotParams, + newPlotParams, bookkeepingParams ); plotFunction(result); diff --git a/apps/ensemble/server/dataFunctions/data_dieoff.js b/apps/ensemble/server/dataFunctions/data_dieoff.js index 96d6301276..fe415261b3 100644 --- a/apps/ensemble/server/dataFunctions/data_dieoff.js +++ b/apps/ensemble/server/dataFunctions/data_dieoff.js @@ -13,6 +13,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataDieoff = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -23,54 +24,77 @@ dataDieoff = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - let error = ""; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; - let statType; - const dataset = []; - const utcCycleStarts = []; + const axisMap = Object.create(null); let xmax = -1 * Number.MAX_VALUE; let ymax = -1 * Number.MAX_VALUE; let xmin = Number.MAX_VALUE; let ymin = Number.MAX_VALUE; + + let statType; + const utcCycleStarts = []; const idealValues = []; + let statement = ""; + let error = ""; + const dataset = []; + for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const statisticSelect = curve.statistic; - const statisticOptionsMap = matsCollections.statistic.findOne( - { name: "statistic" }, - { optionsMap: 1 } - ).optionsMap[appParams.plotType]; - const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + const { threshold } = curve; const thresholdClause = `and m0.trsh = ${threshold}`; + const { members } = curve; const memberClause = `and m0.mem = ${members}`; + const neighborhoodSize = curve["neighborhood-size"]; const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + let validTimeClause = ""; + let validTimes; + + let utcCycleStartClause = ""; + let utcCycleStart; + + const forecastLengthClause = ""; + const forecastLengthStr = curve["dieoff-type"]; + const forecastLengthOptionsMap = matsCollections["dieoff-type"].findOne( + { name: "dieoff-type" }, + { optionsMap: 1 } + ).optionsMap; + const forecastLength = forecastLengthOptionsMap[forecastLengthStr][0]; + + const statisticSelect = curve.statistic; + const statisticOptionsMap = matsCollections.statistic.findOne( + { name: "statistic" }, + { optionsMap: 1 } + ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; + const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; + + const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); + const fromSecs = dateRange.fromSeconds; + const toSecs = dateRange.toSeconds; + let dateClause; + let kernelClause = ""; let probBinClause = ""; let radiusClause = ""; @@ -88,21 +112,18 @@ dataDieoff = function (plotParams, plotFunction) { const { radius } = curve; radiusClause = `and m0.radius = ${radius}`; } - let validTimes; - let validTimeClause = ""; - let utcCycleStart; - let utcCycleStartClause = ""; - const forecastLengthStr = curve["dieoff-type"]; - const forecastLengthOptionsMap = matsCollections["dieoff-type"].findOne( - { name: "dieoff-type" }, - { optionsMap: 1 } - ).optionsMap; - const forecastLength = forecastLengthOptionsMap[forecastLengthStr][0]; - const forecastLengthClause = ""; - const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); - const fromSecs = dateRange.fromSeconds; - const toSecs = dateRange.toSeconds; - let dateClause; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + if (forecastLength === matsTypes.ForecastTypes.dieoff) { validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { @@ -119,7 +140,7 @@ dataDieoff = function (plotParams, plotFunction) { } else { dateClause = `and m0.time-m0.fcst_len*3600 = ${fromSecs}`; } - const [statisticClause] = statisticOptionsMap[statisticSelect]; + // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -134,56 +155,56 @@ dataDieoff = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select m0.fcst_len as fcst_lead, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "{{utcCycleStartClause}} " + - "group by fcst_lead " + - "order by fcst_lead" + - ";"; - - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{utcCycleStartClause}}", utcCycleStartClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select m0.fcst_len as fcst_lead, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "{{utcCycleStartClause}} " + + "group by fcst_lead " + + "order by fcst_lead" + + ";"; + + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{utcCycleStartClause}}", utcCycleStartClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBSpecialtyCurve( - sumPool, + sumPool, // eslint-disable-line no-undef statement, appParams, statisticSelect ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -199,6 +220,7 @@ dataDieoff = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition diff --git a/apps/ensemble/server/dataFunctions/data_gridscale_prob.js b/apps/ensemble/server/dataFunctions/data_gridscale_prob.js index 6ddfd0c9e9..0b75edc4a2 100644 --- a/apps/ensemble/server/dataFunctions/data_gridscale_prob.js +++ b/apps/ensemble/server/dataFunctions/data_gridscale_prob.js @@ -13,6 +13,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataGridScaleProb = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -23,66 +24,84 @@ dataGridScaleProb = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - let error = ""; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; - let statType; - const dataset = []; - const utcCycleStarts = []; + const axisMap = Object.create(null); let xmax = -1 * Number.MAX_VALUE; let ymax = -1 * Number.MAX_VALUE; let xmin = Number.MAX_VALUE; let ymin = Number.MAX_VALUE; + + let statType; + const utcCycleStarts = []; const idealValues = []; + let statement = ""; + let error = ""; + const dataset = []; + for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const statisticSelect = "Grid Scale Count"; - const tableStatPrefix = "count"; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + const { threshold } = curve; const thresholdClause = `and m0.trsh = ${threshold}`; + const { members } = curve; const memberClause = `and m0.mem = ${members}`; + const neighborhoodSize = curve["neighborhood-size"]; const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; - let kernelClause = ""; - const { kernel } = curve; - kernelClause = `and m0.kernel = ${kernel}`; + let validTimeClause = ""; const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; } + const forecastLength = curve["forecast-length"]; const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; + + const statisticSelect = "Grid Scale Count"; + const statisticClause = + "sum(m0.nhdfcstcount) as stat, group_concat(m0.time, ';', m0.nhdfcstcount order by m0.time) as sub_data, count(m0.nhdfcstcount) as N0"; + const tableStatPrefix = "count"; + const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); const fromSecs = dateRange.fromSeconds; const toSecs = dateRange.toSeconds; const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; - const statisticClause = - "sum(m0.nhdfcstcount) as stat, group_concat(m0.time, ';', m0.nhdfcstcount order by m0.time) as sub_data, count(m0.nhdfcstcount) as N0"; + + let kernelClause = ""; + const { kernel } = curve; + kernelClause = `and m0.kernel = ${kernel}`; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -93,50 +112,50 @@ dataGridScaleProb = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select m0.prob as binValue, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by binValue " + - "order by binValue" + - ";"; - - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select m0.prob as binValue, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by binValue " + + "order by binValue" + + ";"; + + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBSpecialtyCurve( - sumPool, + sumPool, // eslint-disable-line no-undef statement, appParams, statisticSelect ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -152,6 +171,7 @@ dataGridScaleProb = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition @@ -224,6 +244,7 @@ dataGridScaleProb = function (plotParams, plotFunction) { // we found no data for any curves so don't bother proceeding throw new Error("INFO: No valid data for any curves."); } + // process the data returned by the query const curveInfoParams = { curves, diff --git a/apps/ensemble/server/dataFunctions/data_histogram.js b/apps/ensemble/server/dataFunctions/data_histogram.js index f8bf5c73c7..ec871d0efc 100644 --- a/apps/ensemble/server/dataFunctions/data_histogram.js +++ b/apps/ensemble/server/dataFunctions/data_histogram.js @@ -11,6 +11,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataHistogram = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -21,20 +22,25 @@ dataHistogram = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; - const alreadyMatched = false; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries const dataFoundForCurve = []; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - let error = ""; + const alreadyMatched = false; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; + + const axisMap = Object.create(null); let statType; let varUnits; + + let statement = ""; + let error = ""; const dataset = []; const allReturnedSubStats = []; const allReturnedSubSecs = []; - const axisMap = Object.create(null); // process user bin customizations const binParams = matsDataUtils.setHistogramParameters(plotParams); @@ -44,35 +50,47 @@ dataHistogram = function (plotParams, plotFunction) { for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; dataFoundForCurve[curveIndex] = true; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const statisticSelect = curve.statistic; - const statisticOptionsMap = matsCollections.statistic.findOne( - { name: "statistic" }, - { optionsMap: 1 } - ).optionsMap[appParams.plotType]; - const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + const { threshold } = curve; const thresholdClause = `and m0.trsh = ${threshold}`; + const { members } = curve; const memberClause = `and m0.mem = ${members}`; + const neighborhoodSize = curve["neighborhood-size"]; const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + let validTimeClause = ""; + const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; + if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { + validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; + } + + const forecastLength = curve["forecast-length"]; + const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; + + const statisticSelect = curve.statistic; + const statisticOptionsMap = matsCollections.statistic.findOne( + { name: "statistic" }, + { optionsMap: 1 } + ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; + const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; + + const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); + const fromSecs = dateRange.fromSeconds; + const toSecs = dateRange.toSeconds; + const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; + let kernelClause = ""; let probBinClause = ""; let radiusClause = ""; @@ -90,18 +108,18 @@ dataHistogram = function (plotParams, plotFunction) { const { radius } = curve; radiusClause = `and m0.radius = ${radius}`; } - let validTimeClause = ""; - const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; - if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { - validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; - } - const forecastLength = curve["forecast-length"]; - const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; - const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); - const fromSecs = dateRange.fromSeconds; - const toSecs = dateRange.toSeconds; - const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; - const [statisticClause] = statisticOptionsMap[statisticSelect]; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -117,54 +135,54 @@ dataHistogram = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select m0.time as avtime, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by avtime " + - "order by avtime" + - ";"; - - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select m0.time as avtime, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by avtime " + + "order by avtime" + + ";"; + + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBSpecialtyCurve( - sumPool, + sumPool, // eslint-disable-line no-undef statement, appParams, statisticSelect ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -182,6 +200,7 @@ dataHistogram = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition diff --git a/apps/ensemble/server/dataFunctions/data_reliability.js b/apps/ensemble/server/dataFunctions/data_reliability.js index 1c788a265f..6fbb4e823c 100644 --- a/apps/ensemble/server/dataFunctions/data_reliability.js +++ b/apps/ensemble/server/dataFunctions/data_reliability.js @@ -12,6 +12,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataReliability = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -22,65 +23,84 @@ dataReliability = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - const dateRange = matsDataUtils.getDateRange(plotParams.dates); - const fromSecs = dateRange.fromSeconds; - const toSecs = dateRange.toSeconds; - let error = ""; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; - let statType; - const dataset = []; + const axisMap = Object.create(null); let xmax = -1 * Number.MAX_VALUE; let ymax = -1 * Number.MAX_VALUE; let xmin = Number.MAX_VALUE; let ymin = Number.MAX_VALUE; + let statType; + + let statement = ""; + let error = ""; + const dataset = []; + + const dateRange = matsDataUtils.getDateRange(plotParams.dates); + const fromSecs = dateRange.fromSeconds; + const toSecs = dateRange.toSeconds; + for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const tableStatPrefix = "count"; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + const { threshold } = curve; const thresholdClause = `and m0.trsh = ${threshold}`; + const { members } = curve; const memberClause = `and m0.mem = ${members}`; + const neighborhoodSize = curve["neighborhood-size"]; const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; - let kernelClause = ""; - const { kernel } = curve; - kernelClause = `and m0.kernel in (0, ${kernel})`; + let validTimeClause = ""; const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; } + const forecastLength = curve["forecast-length"]; const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; - const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; + const statisticClause = "sum(m0.nhdfcstcount) as fcstcount, sum(m0.fcstcount) as rawfcstcount, sum(m0.nhdhitcount) " + "as hitcount, group_concat(m0.time, ';', m0.nhdfcstcount, ';', " + "m0.fcstcount, ';', m0.nhdhitcount order by m0.time) as sub_data, count(m0.nhdfcstcount) as N0"; + const tableStatPrefix = "count"; + + const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; + + let kernelClause = ""; + const { kernel } = curve; + kernelClause = `and m0.kernel in (0, ${kernel})`; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -91,50 +111,50 @@ dataReliability = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select m0.prob as binValue, m0.kernel, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by binValue, kernel " + - "order by binValue, kernel" + - ";"; - - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select m0.prob as binValue, m0.kernel, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by binValue, kernel " + + "order by binValue, kernel" + + ";"; + + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBReliability( - sumPool, + sumPool, // eslint-disable-line no-undef statement, appParams, kernel ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -150,6 +170,7 @@ dataReliability = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition diff --git a/apps/ensemble/server/dataFunctions/data_series.js b/apps/ensemble/server/dataFunctions/data_series.js index b865047870..51c2759f85 100644 --- a/apps/ensemble/server/dataFunctions/data_series.js +++ b/apps/ensemble/server/dataFunctions/data_series.js @@ -13,6 +13,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataSeries = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -23,57 +24,80 @@ dataSeries = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - const dateRange = matsDataUtils.getDateRange(plotParams.dates); - const fromSecs = dateRange.fromSeconds; - const toSecs = dateRange.toSeconds; - let error = ""; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; - let statType; - const dataset = []; - const utcCycleStarts = []; + const axisMap = Object.create(null); let xmax = -1 * Number.MAX_VALUE; let ymax = -1 * Number.MAX_VALUE; let xmin = Number.MAX_VALUE; let ymin = Number.MAX_VALUE; + + let statType; + const utcCycleStarts = []; const idealValues = []; + let statement = ""; + let error = ""; + const dataset = []; + + const dateRange = matsDataUtils.getDateRange(plotParams.dates); + const fromSecs = dateRange.fromSeconds; + const toSecs = dateRange.toSeconds; + for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const statisticSelect = curve.statistic; - const statisticOptionsMap = matsCollections.statistic.findOne( - { name: "statistic" }, - { optionsMap: 1 } - ).optionsMap[appParams.plotType]; - const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + const { threshold } = curve; const thresholdClause = `and m0.trsh = ${threshold}`; + const { members } = curve; const memberClause = `and m0.mem = ${members}`; + const neighborhoodSize = curve["neighborhood-size"]; const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + let validTimeClause = ""; + const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; + if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { + validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; + } + + let forecastLength = curve["forecast-length"]; + const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; + + const statisticSelect = curve.statistic; + const statisticOptionsMap = matsCollections.statistic.findOne( + { name: "statistic" }, + { optionsMap: 1 } + ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; + const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; + + const averageStr = curve.average; + const averageOptionsMap = matsCollections.average.findOne( + { name: "average" }, + { optionsMap: 1 } + ).optionsMap; + const average = averageOptionsMap[averageStr][0]; + + const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; + let kernelClause = ""; let probBinClause = ""; let radiusClause = ""; @@ -91,21 +115,18 @@ dataSeries = function (plotParams, plotFunction) { const { radius } = curve; radiusClause = `and m0.radius = ${radius}`; } - let validTimeClause = ""; - const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; - if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { - validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; - } - let forecastLength = curve["forecast-length"]; - const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; - const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; - const averageStr = curve.average; - const averageOptionsMap = matsCollections.average.findOne( - { name: "average" }, - { optionsMap: 1 } - ).optionsMap; - const average = averageOptionsMap[averageStr][0]; - const [statisticClause] = statisticOptionsMap[statisticSelect]; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -120,54 +141,53 @@ dataSeries = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select {{average}} as avtime, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by avtime " + - "order by avtime" + - ";"; - - statement = statement.replace("{{average}}", average); - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - // math is done on forecastLength later on -- set all analyses to 0 - if (forecastLength === "-99") { - forecastLength = "0"; - } - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select {{average}} as avtime, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by avtime " + + "order by avtime" + + ";"; + + statement = statement.replace("{{average}}", average); + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + + // math is done on forecastLength later on -- set all analyses to 0 + if (forecastLength === "-99") { + forecastLength = "0"; + } + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBTimeSeries( - sumPool, + sumPool, // eslint-disable-line no-undef statement, model, forecastLength, @@ -179,7 +199,9 @@ dataSeries = function (plotParams, plotFunction) { appParams, false ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -195,6 +217,7 @@ dataSeries = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition diff --git a/apps/ensemble/server/dataFunctions/data_threshold.js b/apps/ensemble/server/dataFunctions/data_threshold.js index e99c2c94fd..fb270ad6bb 100644 --- a/apps/ensemble/server/dataFunctions/data_threshold.js +++ b/apps/ensemble/server/dataFunctions/data_threshold.js @@ -13,6 +13,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataThreshold = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -23,52 +24,69 @@ dataThreshold = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - let error = ""; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; - let statType; - const dataset = []; - const utcCycleStarts = []; + const axisMap = Object.create(null); let xmax = -1 * Number.MAX_VALUE; let ymax = -1 * Number.MAX_VALUE; let xmin = Number.MAX_VALUE; let ymin = Number.MAX_VALUE; + + let statType; + const utcCycleStarts = []; const idealValues = []; + let statement = ""; + let error = ""; + const dataset = []; + for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const { members } = curve; + const memberClause = `and m0.mem = ${members}`; + + const neighborhoodSize = curve["neighborhood-size"]; + const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + let validTimeClause = ""; + const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; + if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { + validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; + } + + const forecastLength = curve["forecast-length"]; + const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; + const statisticSelect = curve.statistic; const statisticOptionsMap = matsCollections.statistic.findOne( { name: "statistic" }, { optionsMap: 1 } ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; - const { members } = curve; - const memberClause = `and m0.mem = ${members}`; - const neighborhoodSize = curve["neighborhood-size"]; - const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); + const fromSecs = dateRange.fromSeconds; + const toSecs = dateRange.toSeconds; + const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; + let kernelClause = ""; let probBinClause = ""; let radiusClause = ""; @@ -86,18 +104,18 @@ dataThreshold = function (plotParams, plotFunction) { const { radius } = curve; radiusClause = `and m0.radius = ${radius}`; } - let validTimeClause = ""; - const validTimes = curve["valid-time"] === undefined ? [] : curve["valid-time"]; - if (validTimes.length !== 0 && validTimes !== matsTypes.InputTypes.unused) { - validTimeClause = `and floor((m0.time)%(24*3600)/3600) IN(${validTimes})`; - } - const forecastLength = curve["forecast-length"]; - const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; - const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); - const fromSecs = dateRange.fromSeconds; - const toSecs = dateRange.toSeconds; - const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; - const [statisticClause] = statisticOptionsMap[statisticSelect]; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -112,52 +130,52 @@ dataThreshold = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select m0.trsh as thresh, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{validTimeClause}} " + - "{{forecastLengthClause}} " + - "group by thresh " + - "order by thresh" + - ";"; - - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{validTimeClause}}", validTimeClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select m0.trsh as thresh, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{validTimeClause}} " + + "{{forecastLengthClause}} " + + "group by thresh " + + "order by thresh" + + ";"; + + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{validTimeClause}}", validTimeClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBSpecialtyCurve( - sumPool, + sumPool, // eslint-disable-line no-undef statement, appParams, statisticSelect ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -173,6 +191,7 @@ dataThreshold = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition diff --git a/apps/ensemble/server/dataFunctions/data_validtime.js b/apps/ensemble/server/dataFunctions/data_validtime.js index 48f91a1eaf..a8ad24039b 100644 --- a/apps/ensemble/server/dataFunctions/data_validtime.js +++ b/apps/ensemble/server/dataFunctions/data_validtime.js @@ -13,6 +13,7 @@ import { } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +// eslint-disable-next-line no-undef dataValidTime = function (plotParams, plotFunction) { // initialize variables common to all curves const appParams = { @@ -23,54 +24,66 @@ dataValidTime = function (plotParams, plotFunction) { hideGaps: plotParams.noGapsCheck, hasLevels: false, }; + + const totalProcessingStart = moment(); const dataRequests = {}; // used to store data queries let dataFoundForCurve = true; let dataFoundForAnyCurve = false; - const totalProcessingStart = moment(); - let error = ""; + const curves = JSON.parse(JSON.stringify(plotParams.curves)); const curvesLength = curves.length; - let statType; - const dataset = []; - const utcCycleStarts = []; + const axisMap = Object.create(null); let xmax = -1 * Number.MAX_VALUE; let ymax = -1 * Number.MAX_VALUE; let xmin = Number.MAX_VALUE; let ymin = Number.MAX_VALUE; + + let statType; + const utcCycleStarts = []; const idealValues = []; + let statement = ""; + let error = ""; + const dataset = []; + for (let curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // initialize variables specific to each curve const curve = curves[curveIndex]; - const { diffFrom } = curve; const { label } = curve; + const { diffFrom } = curve; + const { variable } = curve; const databaseRef = matsCollections.variable.findOne({ name: "variable" }) .optionsMap[variable]; const model = matsCollections["data-source"].findOne({ name: "data-source" }) .optionsMap[variable][curve["data-source"]][0]; - const regionStr = curve.region; - let region = Object.keys( - matsCollections.region.findOne({ name: "region" }).valuesMap - ).find( - (key) => - matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr - ); - region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do - const statisticSelect = curve.statistic; - const statisticOptionsMap = matsCollections.statistic.findOne( - { name: "statistic" }, - { optionsMap: 1 } - ).optionsMap[appParams.plotType]; - const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; - const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; + const { threshold } = curve; const thresholdClause = `and m0.trsh = ${threshold}`; + const { members } = curve; const memberClause = `and m0.mem = ${members}`; + const neighborhoodSize = curve["neighborhood-size"]; const neighborhoodClause = `and m0.nhd_size = ${neighborhoodSize}`; + + const forecastLength = curve["forecast-length"]; + const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; + + const statisticSelect = curve.statistic; + const statisticOptionsMap = matsCollections.statistic.findOne( + { name: "statistic" }, + { optionsMap: 1 } + ).optionsMap[appParams.plotType]; + const [statisticClause] = statisticOptionsMap[statisticSelect]; + const tableStatPrefix = statisticOptionsMap[statisticSelect][2]; + + const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); + const fromSecs = dateRange.fromSeconds; + const toSecs = dateRange.toSeconds; + const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; + let kernelClause = ""; let probBinClause = ""; let radiusClause = ""; @@ -88,13 +101,17 @@ dataValidTime = function (plotParams, plotFunction) { const { radius } = curve; radiusClause = `and m0.radius = ${radius}`; } - const forecastLength = curve["forecast-length"]; - const forecastLengthClause = `and m0.fcst_len = ${forecastLength}`; - const dateRange = matsDataUtils.getDateRange(curve["curve-dates"]); - const fromSecs = dateRange.fromSeconds; - const toSecs = dateRange.toSeconds; - const dateClause = `and m0.time >= ${fromSecs} and m0.time <= ${toSecs}`; - const [statisticClause] = statisticOptionsMap[statisticSelect]; + + const regionStr = curve.region; + let region = Object.keys( + matsCollections.region.findOne({ name: "region" }).valuesMap + ).find( + (key) => + matsCollections.region.findOne({ name: "region" }).valuesMap[key] === regionStr + ); + region = region === "Full" ? "Full_domain" : region; // this db doesn't handle the full domain the way the others do + + const queryTableClause = `from ${databaseRef}.${model}_${tableStatPrefix}_${region} as m0`; // axisKey is used to determine which axis a curve should use. // This axisKeySet object is used like a set and if a curve has the same // units (axisKey) it will use the same axis. @@ -109,52 +126,52 @@ dataValidTime = function (plotParams, plotFunction) { let d; if (!diffFrom) { - // this is a database driven curve, not a difference curve - // prepare the query from the above parameters - let statement = - "select floor(m0.time%(24*3600)/3600) as hr_of_day, " + - "count(distinct m0.time) as N_times, " + - "min(m0.time) as min_secs, " + - "max(m0.time) as max_secs, " + - "{{statisticClause}} " + - "{{queryTableClause}} " + - "where 1=1 " + - "{{dateClause}} " + - "{{memberClause}} " + - "{{neighborhoodClause}} " + - "{{thresholdClause}} " + - "{{kernelClause}} " + - "{{probBinClause}} " + - "{{radiusClause}} " + - "{{forecastLengthClause}} " + - "group by hr_of_day " + - "order by hr_of_day" + - ";"; - - statement = statement.replace("{{statisticClause}}", statisticClause); - statement = statement.replace("{{queryTableClause}}", queryTableClause); - statement = statement.replace("{{memberClause}}", memberClause); - statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); - statement = statement.replace("{{thresholdClause}}", thresholdClause); - statement = statement.replace("{{kernelClause}}", kernelClause); - statement = statement.replace("{{probBinClause}}", probBinClause); - statement = statement.replace("{{radiusClause}}", radiusClause); - statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); - statement = statement.replace("{{dateClause}}", dateClause); - dataRequests[label] = statement; - let queryResult; const startMoment = moment(); let finishMoment; try { + statement = + "select floor(m0.time%(24*3600)/3600) as hr_of_day, " + + "count(distinct m0.time) as N_times, " + + "min(m0.time) as min_secs, " + + "max(m0.time) as max_secs, " + + "{{statisticClause}} " + + "{{queryTableClause}} " + + "where 1=1 " + + "{{dateClause}} " + + "{{memberClause}} " + + "{{neighborhoodClause}} " + + "{{thresholdClause}} " + + "{{kernelClause}} " + + "{{probBinClause}} " + + "{{radiusClause}} " + + "{{forecastLengthClause}} " + + "group by hr_of_day " + + "order by hr_of_day" + + ";"; + + statement = statement.replace("{{statisticClause}}", statisticClause); + statement = statement.replace("{{queryTableClause}}", queryTableClause); + statement = statement.replace("{{memberClause}}", memberClause); + statement = statement.replace("{{neighborhoodClause}}", neighborhoodClause); + statement = statement.replace("{{thresholdClause}}", thresholdClause); + statement = statement.replace("{{kernelClause}}", kernelClause); + statement = statement.replace("{{probBinClause}}", probBinClause); + statement = statement.replace("{{radiusClause}}", radiusClause); + statement = statement.replace("{{forecastLengthClause}}", forecastLengthClause); + statement = statement.replace("{{dateClause}}", dateClause); + dataRequests[label] = statement; + // send the query statement to the query function queryResult = matsDataQueryUtils.queryDBSpecialtyCurve( - sumPool, + sumPool, // eslint-disable-line no-undef statement, appParams, statisticSelect ); + finishMoment = moment(); + dataRequests[label] = statement; dataRequests[`data retrieval (query) time - ${label}`] = { begin: startMoment.format(), finish: finishMoment.format(), @@ -170,6 +187,7 @@ dataValidTime = function (plotParams, plotFunction) { e.message = `Error in queryDB: ${e.message} for statement: ${statement}`; throw new Error(e.message); } + if (queryResult.error !== undefined && queryResult.error !== "") { if (queryResult.error === matsTypes.Messages.NO_DATA_FOUND) { // this is NOT an error just a no data condition diff --git a/apps/ensemble/server/main.js b/apps/ensemble/server/main.js index 1f2ffa9b8a..7d797642ba 100644 --- a/apps/ensemble/server/main.js +++ b/apps/ensemble/server/main.js @@ -5,6 +5,7 @@ import { Meteor } from "meteor/meteor"; import { mysql } from "meteor/pcel:mysql"; import { moment } from "meteor/momentjs:moment"; +import { _ } from "meteor/underscore"; import { matsMethods, matsTypes, @@ -328,42 +329,36 @@ const doCurveParams = function () { const thresholdsModelOptionsMap = {}; const kernelModelOptionsMap = {}; const radiusModelOptionsMap = {}; - const masterRegionValuesMap = {}; + const allRegionValuesMap = {}; let allKernels = []; try { const rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( - metadataPool, + metadataPool, // eslint-disable-line no-undef "select short_name,description from region_descriptions;" ); - let masterRegDescription; - let masterShortName; for (let j = 0; j < rows.length; j += 1) { - masterRegDescription = rows[j].description.trim(); - masterShortName = rows[j].short_name.trim(); - masterRegionValuesMap[masterShortName] = masterRegDescription; + allRegionValuesMap[rows[j].short_name.trim()] = rows[j].description.trim(); } } catch (err) { throw new Error(err.message); } - let rows; - let didx; - try { - for (didx = 0; didx < variables.length; didx += 1) { - modelOptionsMap[variables[didx]] = {}; - modelDateRangeMap[variables[didx]] = {}; - forecastLengthOptionsMap[variables[didx]] = {}; - memberModelOptionsMap[variables[didx]] = {}; - neighborhoodModelOptionsMap[variables[didx]] = {}; - thresholdsModelOptionsMap[variables[didx]] = {}; - kernelModelOptionsMap[variables[didx]] = {}; - radiusModelOptionsMap[variables[didx]] = {}; - regionModelOptionsMap[variables[didx]] = {}; + for (let didx = 0; didx < variables.length; didx += 1) { + const variable = variables[didx]; + modelOptionsMap[variable] = {}; + modelDateRangeMap[variable] = {}; + forecastLengthOptionsMap[variable] = {}; + memberModelOptionsMap[variable] = {}; + neighborhoodModelOptionsMap[variable] = {}; + thresholdsModelOptionsMap[variable] = {}; + kernelModelOptionsMap[variable] = {}; + radiusModelOptionsMap[variable] = {}; + regionModelOptionsMap[variable] = {}; - rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( - sumPool, + const rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( + sumPool, // eslint-disable-line no-undef `select model,regions,display_text,fcst_lens,mems,nhd_sizes,trshs,kernels,radii,mindate,maxdate from ${ variableDBNames[variables[didx]] }.regions_per_model_mats_all_categories order by display_category, display_order;` @@ -371,7 +366,7 @@ const doCurveParams = function () { for (let i = 0; i < rows.length; i += 1) { const modelValue = rows[i].model.trim(); const model = rows[i].display_text.trim(); - modelOptionsMap[variables[didx]][model] = [modelValue]; + modelOptionsMap[variable][model] = [modelValue]; const rowMinDate = moment .utc(rows[i].mindate * 1000) @@ -379,82 +374,73 @@ const doCurveParams = function () { const rowMaxDate = moment .utc(rows[i].maxdate * 1000) .format("MM/DD/YYYY HH:mm"); - modelDateRangeMap[variables[didx]][model] = { + modelDateRangeMap[variable][model] = { minDate: rowMinDate, maxDate: rowMaxDate, }; const forecastLengths = rows[i].fcst_lens; - const forecastLengthArr = forecastLengths + forecastLengthOptionsMap[variable][model] = forecastLengths .split(",") - .map(Function.prototype.call, String.prototype.trim); - for (let j = 0; j < forecastLengthArr.length; j += 1) { - forecastLengthArr[j] = forecastLengthArr[j].replace(/'|\[|\]/g, ""); - } - forecastLengthOptionsMap[variables[didx]][model] = forecastLengthArr; + .map(Function.prototype.call, String.prototype.trim) + .map(function (fhr) { + return fhr.replace(/'|\[|\]/g, ""); + }); const { mems } = rows[i]; - const memArr = mems + memberModelOptionsMap[variable][model] = mems .split(",") - .map(Function.prototype.call, String.prototype.trim); - for (let j = 0; j < memArr.length; j += 1) { - memArr[j] = memArr[j].replace(/'|\[|\]/g, ""); - } - memberModelOptionsMap[variables[didx]][model] = memArr; + .map(Function.prototype.call, String.prototype.trim) + .map(function (mem) { + return mem.replace(/'|\[|\]/g, ""); + }); const nhdSizes = rows[i].nhd_sizes; - const nhdSizeArr = nhdSizes + neighborhoodModelOptionsMap[variable][model] = nhdSizes .split(",") - .map(Function.prototype.call, String.prototype.trim); - for (let j = 0; j < nhdSizeArr.length; j += 1) { - nhdSizeArr[j] = nhdSizeArr[j].replace(/'|\[|\]/g, ""); - } - neighborhoodModelOptionsMap[variables[didx]][model] = nhdSizeArr; + .map(Function.prototype.call, String.prototype.trim) + .map(function (nhdSize) { + return nhdSize.replace(/'|\[|\]/g, ""); + }); const thresholds = rows[i].trshs; - const thresholdsArr = thresholds + thresholdsModelOptionsMap[variable][model] = thresholds .split(",") - .map(Function.prototype.call, String.prototype.trim); - for (let j = 0; j < thresholdsArr.length; j += 1) { - thresholdsArr[j] = thresholdsArr[j].replace(/'|\[|\]/g, ""); - } - thresholdsModelOptionsMap[variables[didx]][model] = thresholdsArr; + .map(Function.prototype.call, String.prototype.trim) + .map(function (threshold) { + return threshold.replace(/'|\[|\]/g, ""); + }); const { kernels } = rows[i]; - const kernelArr = kernels + kernelModelOptionsMap[variable][model] = kernels .split(",") - .map(Function.prototype.call, String.prototype.trim); - for (let j = 0; j < kernelArr.length; j += 1) { - kernelArr[j] = kernelArr[j].replace(/'|\[|\]/g, ""); - } - kernelModelOptionsMap[variables[didx]][model] = kernelArr; - allKernels = _.union(allKernels, kernelArr); + .map(Function.prototype.call, String.prototype.trim) + .map(function (kernel) { + return kernel.replace(/'|\[|\]/g, ""); + }); + allKernels = _.union(allKernels, kernelModelOptionsMap[variable][model]); const { radii } = rows[i]; - const radiusArr = radii + radiusModelOptionsMap[variable][model] = radii .split(",") - .map(Function.prototype.call, String.prototype.trim); - for (let j = 0; j < radiusArr.length; j += 1) { - radiusArr[j] = radiusArr[j].replace(/'|\[|\]/g, ""); - } - radiusModelOptionsMap[variables[didx]][model] = radiusArr; + .map(Function.prototype.call, String.prototype.trim) + .map(function (radius) { + return radius.replace(/'|\[|\]/g, ""); + }); const { regions } = rows[i]; - const regionsArrRaw = regions + regionModelOptionsMap[variable][model] = regions .split(",") - .map(Function.prototype.call, String.prototype.trim); - const regionsArr = []; - let dummyRegion; - for (let j = 0; j < regionsArrRaw.length; j += 1) { - dummyRegion = regionsArrRaw[j].replace(/'|\[|\]/g, ""); - regionsArr.push(masterRegionValuesMap[dummyRegion]); - } - regionModelOptionsMap[variables[didx]][model] = regionsArr; + .map(Function.prototype.call, String.prototype.trim) + .map(function (region) { + return allRegionValuesMap[region.replace(/'|\[|\]/g, "")]; + }); } } } catch (err) { throw new Error(err.message); } + if (matsCollections.label.findOne({ name: "label" }) === undefined) { matsCollections.label.insert({ name: "label", @@ -559,7 +545,7 @@ const doCurveParams = function () { regionModelOptionsMap[variables[0]][ Object.keys(regionModelOptionsMap[variables[0]])[0] ], - valuesMap: masterRegionValuesMap, + valuesMap: allRegionValuesMap, superiorNames: ["variable", "data-source"], controlButtonCovered: true, unique: false, @@ -577,7 +563,7 @@ const doCurveParams = function () { const currentParam = matsCollections.region.findOne({ name: "region" }); if ( !matsDataUtils.areObjectsEqual(currentParam.optionsMap, regionModelOptionsMap) || - !matsDataUtils.areObjectsEqual(currentParam.valuesMap, masterRegionValuesMap) + !matsDataUtils.areObjectsEqual(currentParam.valuesMap, allRegionValuesMap) ) { // have to reload region data matsCollections.region.update( @@ -585,7 +571,7 @@ const doCurveParams = function () { { $set: { optionsMap: regionModelOptionsMap, - valuesMap: masterRegionValuesMap, + valuesMap: allRegionValuesMap, options: regionModelOptionsMap[variables[0]][ Object.keys(regionModelOptionsMap[variables[0]])[0] @@ -1103,7 +1089,7 @@ const doCurveParams = function () { const probBinOptionsMap = { 0: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], }; - for (didx = 0; didx < allKernels.length; didx += 1) { + for (let didx = 0; didx < allKernels.length; didx += 1) { probBinOptionsMap[allKernels[didx]] = [ "0", "10", @@ -1672,6 +1658,7 @@ const doPlotGraph = function () { Meteor.startup(function () { matsCollections.Databases.remove({}); if (matsCollections.Databases.find({}).count() < 0) { + // eslint-disable-next-line no-console console.warn( "main startup: corrupted Databases collection: dropping Databases collection" ); @@ -1716,6 +1703,7 @@ Meteor.startup(function () { ); if (cbConnection) { // global cbScorecardSettingsPool + // eslint-disable-next-line no-undef cbScorecardSettingsPool = new matsCouchbaseUtils.CBUtilities( cbConnection.host, cbConnection.bucket, @@ -1742,6 +1730,7 @@ Meteor.startup(function () { ); // the pool is intended to be global if (metadataSettings) { + // eslint-disable-next-line no-undef metadataPool = mysql.createPool(metadataSettings); allPools.push({ pool: "metadataPool", role: matsTypes.DatabaseRoles.META_DATA }); } @@ -1762,6 +1751,7 @@ Meteor.startup(function () { ); // the pool is intended to be global if (sumSettings) { + // eslint-disable-next-line no-undef sumPool = mysql.createPool(sumSettings); allPools.push({ pool: "sumPool", role: matsTypes.DatabaseRoles.SUMS_DATA }); } @@ -1790,6 +1780,7 @@ Meteor.startup(function () { // These are application specific mongo data - like curve params // The appSpecificResetRoutines object is a special name, // as is doCurveParams. The refreshMetaData mechanism depends on them being named that way. +// eslint-disable-next-line no-undef appSpecificResetRoutines = [ doPlotGraph, doCurveParams,