From 8861fbf5f053ddbd655f52e18cc2711a63e3e214 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Thu, 15 Aug 2024 13:53:11 -0600 Subject: [PATCH] Linted matsMethods --- .eslintrc.json | 3 +- .../.npm/package/npm-shrinkwrap.json | 62 +- .../imports/startup/api/matsMethods.js | 7049 +++++++++-------- .../startup/server/data_plot_ops_util.js | 4 +- .../startup/server/data_process_util.js | 68 +- .../imports/startup/server/data_query_util.js | 52 +- .../imports/startup/server/data_util.js | 50 +- meteor_packages/mats-common/package.js | 52 +- 8 files changed, 3701 insertions(+), 3639 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 0d777e35d..3c5c1d052 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -24,7 +24,7 @@ // our dependencies. // XXX: this *should* be taken care of by eslint-import-resolver-meteor, investigate. // "import/no-extraneous-dependencies": "off", - // "no-underscore-dangle": ["error", { "allow": ["_id", "_ensureIndex"] }], + "no-underscore-dangle": ["error", { "allow": ["_id", "_ensureIndex"] }], "object-shorthand": ["error", "always", { "avoidQuotes": false }], "space-before-function-paren": "off", // for Meteor API's that rely on `this` context, e.g. Template.onCreated and publications @@ -51,7 +51,6 @@ "radix": "warn", "global-require": "warn", "no-lonely-if": "warn", - "no-underscore-dangle": ["off", { "allow": ["_id", "_ensureIndex"] }], // Uncomment line 27 with Meteor's recommended "no-underscore-dangle" policy when resolving this "no-return-assign": "warn", "no-shadow": "warn", "new-cap": "warn", diff --git a/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json b/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json index 2cfe6142f..e10d6f950 100644 --- a/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json +++ b/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json @@ -42,9 +42,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "axios": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", - "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==" + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", + "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==" }, "call-bind": { "version": "1.0.7", @@ -61,11 +61,6 @@ "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==" }, - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" - }, "cmake-js": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/cmake-js/-/cmake-js-7.3.0.tgz", @@ -127,9 +122,9 @@ "integrity": "sha512-DxWXvvPq4srWLCqFugqSV+6CBt/CvQ0dnpXhQ3gl0autcIDAruG1PuGG3gC7yPRNytAD1oU1AcUOzaYhOawhTw==" }, "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==" + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==" }, "deep-extend": { "version": "0.6.0", @@ -717,40 +712,40 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==" }, - "mongo-object": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mongo-object/-/mongo-object-3.0.1.tgz", - "integrity": "sha512-EbiwWHvKOF9xhIzuwaqknwPISdkHMipjMs6DiJFicupgBBLEhUs0OOro9MuPkFogB17DZlsV4KJhhxfqZ7ZRMQ==" - }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node-addon-api": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz", - "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==" + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" }, "node-api-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/node-api-headers/-/node-api-headers-1.1.0.tgz", - "integrity": "sha512-ucQW+SbYCUPfprvmzBsnjT034IGRB2XK8rRc78BgjNKhTdFKgAwAmgW704bKIBmcYW48it0Gkjpkd39Azrwquw==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/node-api-headers/-/node-api-headers-1.2.0.tgz", + "integrity": "sha512-L9AiEkBfgupC0D/LsudLPOhzy/EdObsp+FHyL1zSK0kKv5FDA9rJMoRz8xd+ojxzlqfg0tTZm2h8ot2nS7bgRA==" }, "node-file-cache": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/node-file-cache/-/node-file-cache-1.0.2.tgz", "integrity": "sha512-X+u705v8tTdOnNhTkkSdb36QbuILf1MjSmSb8JlrpzLIzlLFu7qmCF3ezaAFPz9w5vFe0NpZfa7kn2v6eW+SAg==" }, + "nodemailer": { + "version": "6.9.14", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.14.tgz", + "integrity": "sha512-Dobp/ebDKBvz91sbtRKhcznLThrKxKt97GI2FAlAyy+fk19j73Uz3sBXolVtmcXjaorivqsbbbjDY+Jkt4/bQA==" + }, "npmlog": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==" }, "object-inspect": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==" }, "proxy-from-env": { "version": "1.1.0", @@ -768,9 +763,9 @@ "integrity": "sha512-RUOOOjHLhgR1MIQrCtnEqz/HJ1RMZBIN+REnpSUrfft2bXqXy69fwJASVziWExfFXsR1bCY0TznnHooNsCo0/w==" }, "qs": { - "version": "6.12.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.12.1.tgz", - "integrity": "sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==" + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==" }, "rc": { "version": "1.2.8", @@ -798,9 +793,9 @@ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==" + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==" }, "set-blocking": { "version": "2.0.0", @@ -822,11 +817,6 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, - "simpl-schema": { - "version": "3.4.6", - "resolved": "https://registry.npmjs.org/simpl-schema/-/simpl-schema-3.4.6.tgz", - "integrity": "sha512-xgShTrNzktC1TTgizSjyDHrxs0bmZa1b9sso54cL8xwO2OloVhtHjfO73/dAK9OFzUIWCBTpKMpD12JPTgVimA==" - }, "steno": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/steno/-/steno-0.4.4.tgz", diff --git a/meteor_packages/mats-common/imports/startup/api/matsMethods.js b/meteor_packages/mats-common/imports/startup/api/matsMethods.js index bb8e5d4e0..38a70de68 100644 --- a/meteor_packages/mats-common/imports/startup/api/matsMethods.js +++ b/meteor_packages/mats-common/imports/startup/api/matsMethods.js @@ -4,7 +4,8 @@ import { Meteor } from "meteor/meteor"; import { ValidatedMethod } from "meteor/mdg:validated-method"; -import SimpleSchema from "simpl-schema"; +import SimpleSchema from "meteor/aldeed:simple-schema"; +import { Picker } from "meteor/meteorhacks:picker"; import { matsCache, matsCollections, @@ -15,10 +16,17 @@ import { versionInfo, } from "meteor/randyp:mats-common"; import { mysql } from "meteor/pcel:mysql"; -import { url } from "url"; +import { moment } from "meteor/momentjs:moment"; +import { _ } from "meteor/underscore"; import { Mongo } from "meteor/mongo"; import { curveParamsByApp } from "../both/mats-curve-params"; +/* global cbPool, cbScorecardPool, cbScorecardSettingsPool, appSpecificResetRoutines */ + +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-console */ +/* eslint-disable global-require */ + // PRIVATE // local collection used to keep the table update times for refresh - won't ever be synchronized or persisted. @@ -30,714 +38,1676 @@ const DownSampleResults = new Mongo.Collection("DownSampleResults"); // utility to check for empty object const isEmpty = function (map) { - for (const key in map) { - if (map.hasOwnProperty(key)) { - return false; - } - } - return true; + const mapKeys = Object.keys(map); + return mapKeys.length === 0; }; -// Define routes for server -if (Meteor.isServer) { - // add indexes to result and axes collections - DownSampleResults.rawCollection().createIndex( - { - createdAt: 1, - }, - { - expireAfterSeconds: 3600 * 8, - } - ); // 8 hour expiration - LayoutStoreCollection.rawCollection().createIndex( - { - createdAt: 1, - }, - { - expireAfterSeconds: 900, - } - ); // 15 min expiration - // set the default proxy prefix path to "" - // If the settings are not complete, they will be set by the configuration and written out, which will cause the app to reset - if (Meteor.settings.public && !Meteor.settings.public.proxy_prefix_path) { - Meteor.settings.public.proxy_prefix_path = ""; +// private middleware for getting the status - think health check +const status = function (res) { + if (Meteor.isServer) { + const settings = matsCollections.Settings.findOne(); + res.end( + `
Running: version - ${settings.appVersion}
` + ); } +}; - Picker.route("/status", function (params, req, res, next) { - Picker.middleware(_status(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/status`, - function (params, req, res, next) { - Picker.middleware(_status(params, req, res, next)); +// private - used to see if the main page needs to update its selectors +const checkMetaDataRefresh = async function () { + // This routine compares the current last modified time of the tables (MYSQL) or documents (Couchbase) + // used for curveParameter metadata with the last update time to determine if an update is necessary. + // We really only do this for Curveparams + /* + metaDataTableUpdates: + { + name: dataBaseName(MYSQL) or bucketName(couchbase), + (for couchbase tables are documents) + tables: [tableName1, tableName2 ..], + lastRefreshed : timestamp + } + */ + let refresh = false; + const tableUpdates = metaDataTableUpdates.find({}).fetch(); + const dbType = + matsCollections.Settings.findOne() !== undefined + ? matsCollections.Settings.findOne().dbType + : matsTypes.DbTypes.mysql; + for (let tui = 0; tui < tableUpdates.length; tui += 1) { + const id = tableUpdates[tui]._id; + const poolName = tableUpdates[tui].pool; + const dbName = tableUpdates[tui].name; + const tableNames = tableUpdates[tui].tables; + const { lastRefreshed } = tableUpdates[tui]; + let updatedEpoch = Number.MAX_VALUE; + let rows; + let doc; + for (let ti = 0; ti < tableNames.length; ti += 1) { + const tName = tableNames[ti]; + try { + if (Meteor.isServer) { + switch (dbType) { + case matsTypes.DbTypes.mysql: + rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( + global[poolName], + `SELECT UNIX_TIMESTAMP(UPDATE_TIME)` + + ` FROM information_schema.tables` + + ` WHERE TABLE_SCHEMA = '${dbName}'` + + ` AND TABLE_NAME = '${tName}'` + ); + updatedEpoch = rows[0]["UNIX_TIMESTAMP(UPDATE_TIME)"]; + break; + case matsTypes.DbTypes.couchbase: + // the tName for couchbase is supposed to be the document id + doc = await cbPool.getCB(tName); + updatedEpoch = doc.updated; + break; + default: + throw new Meteor.Error("resetApp: undefined DbType"); + } + } + // console.log("DB says metadata for table " + dbName + "." + tName + " was updated at " + updatedEpoch); + if ( + updatedEpoch === undefined || + updatedEpoch === null || + updatedEpoch === "NULL" || + updatedEpoch === Number.MAX_VALUE + ) { + // if time of last update isn't stored by the database (thanks, Aurora DB), refresh automatically + // console.log("_checkMetaDataRefresh - cannot find last update time for database: " + dbName + " and table: " + tName); + refresh = true; + // console.log("FORCED Refreshing the metadata for table because updatedEpoch is undefined" + dbName + "." + tName + " : updated at " + updatedEpoch); + break; + } + } catch (e) { + throw new Error( + `_checkMetaDataRefresh - error finding last update time for database: ${dbName} and table: ${tName}, ERROR:${e.message}` + ); + } + const lastRefreshedEpoch = moment.utc(lastRefreshed).valueOf() / 1000; + const updatedEpochMoment = moment.utc(updatedEpoch).valueOf(); + // console.log("Epoch of when this app last refreshed metadata for table " + dbName + "." + tName + " is " + lastRefreshedEpoch); + // console.log("Epoch of when the DB says table " + dbName + "." + tName + " was last updated is " + updatedEpochMoment); + if (lastRefreshedEpoch < updatedEpochMoment || updatedEpochMoment === 0) { + // Aurora DB sometimes returns a 0 for last updated. In that case, do refresh the metadata. + refresh = true; + // console.log("Refreshing the metadata in the app selectors because table " + dbName + "." + tName + " was updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " while the metadata was last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); + break; + } else { + // console.log("NOT Refreshing the metadata for table " + dbName + "." + tName + " : updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " : metadata last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); + } } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/status`, - function (params, req, res, next) { - Picker.middleware(_status(params, req, res, next)); + if (refresh === true) { + // refresh the app metadata + // app specific routines + for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { + await global.appSpecificResetRoutines[ai](); + } + // remember that we updated ALL the metadata tables just now + metaDataTableUpdates.update( + { + _id: id, + }, + { + $set: { + lastRefreshed: moment().format(), + }, + } + ); } - ); - - Picker.route("/_getCSV/:key", function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - }); + } + return true; +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/_getCSV/:key`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - } - ); +// private middleware for clearing the cache +const clearCache = function (res) { + if (Meteor.isServer) { + matsCache.clear(); + res.end("

clearCache Done!

"); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/app:/_getCSV/:key`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); +// private middleware for dropping a distinct instance (a single run) of a scorecard +const dropThisScorecardInstance = async function ( + userName, + name, + submittedTime, + processedAt +) { + try { + if (cbScorecardPool === undefined) { + throw new Meteor.Error("dropThisScorecardInstance: No cbScorecardPool defined"); } - ); - - Picker.route("/CSV/:f/:key/:m/:a", function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - }); + const statement = `DELETE + From + vxdata._default.SCORECARD sc + WHERE + sc.type='SC' + AND sc.userName='${userName}' + AND sc.name='${name}' + AND sc.processedAt=${processedAt} + AND sc.submitted=${submittedTime};`; + return await cbScorecardPool.queryCB(statement); + // delete this result from the mongo Scorecard collection + } catch (err) { + console.log(`dropThisScorecardInstance error : ${err.message}`); + return { + error: err.message, + }; + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/CSV/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - } - ); +// helper function for returning an array of database-distinct apps contained within a larger MATS app +function getListOfApps() { + let apps; + if ( + matsCollections.database !== undefined && + matsCollections.database.findOne({ name: "database" }) !== undefined + ) { + // get list of databases (one per app) + apps = matsCollections.database.findOne({ + name: "database", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + } else if ( + matsCollections.variable !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }) !== undefined && + matsCollections.threshold !== undefined && + matsCollections.threshold.findOne({ + name: "threshold", + }) !== undefined + ) { + // get list of apps (variables in apps that also have thresholds) + apps = matsCollections.variable.findOne({ + name: "variable", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + } else { + apps = [matsCollections.Settings.findOne().Title]; + } + return apps; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/CSV/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); +// helper function to map a results array to specific apps +function mapArrayToApps(result) { + // put results in a map keyed by app + const newResult = {}; + const apps = getListOfApps(); + for (let aidx = 0; aidx < apps.length; aidx += 1) { + if (result[aidx] === apps[aidx]) { + newResult[apps[aidx]] = [result[aidx]]; + } else { + newResult[apps[aidx]] = result; } - ); - - Picker.route("/_getJSON/:key", function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - }); + } + return newResult; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/_getJSON/:key`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); +// helper function to map a results map to specific apps +function mapMapToApps(result) { + // put results in a map keyed by app + let newResult = {}; + let tempResult; + const apps = getListOfApps(); + const resultKeys = Object.keys(result); + if (!matsDataUtils.arraysEqual(apps.sort(), resultKeys.sort())) { + if (resultKeys.includes("Predefined region")) { + tempResult = result["Predefined region"]; + } else { + tempResult = result; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/app:/_getJSON/:key`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); + for (let aidx = 0; aidx < apps.length; aidx += 1) { + newResult[apps[aidx]] = tempResult; } - ); + } else { + newResult = result; + } + return newResult; +} - Picker.route("/JSON/:f/:key/:m/:a", function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/JSON/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/JSON/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - } - ); - - Picker.route("/clearCache", function (params, req, res, next) { - Picker.middleware(_clearCache(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/clearCache`, - function (params, req, res, next) { - Picker.middleware(_clearCache(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/clearCache`, - function (params, req, res, next) { - Picker.middleware(_clearCache(params, req, res, next)); +// helper function for returning a map of database-distinct apps contained within a larger MATS app and their DBs +function getListOfAppDBs() { + let apps; + const result = {}; + let aidx; + if ( + matsCollections.database !== undefined && + matsCollections.database.findOne({ name: "database" }) !== undefined + ) { + // get list of databases (one per app) + apps = matsCollections.database.findOne({ + name: "database", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + for (aidx = 0; aidx < apps.length; aidx += 1) { + result[apps[aidx]] = matsCollections.database.findOne({ + name: "database", + }).optionsMap[apps[aidx]].sumsDB; } - ); - - Picker.route("/getApps", function (params, req, res, next) { - Picker.middleware(_getApps(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getApps`, - function (params, req, res, next) { - Picker.middleware(_getApps(params, req, res, next)); + } else if ( + matsCollections.variable !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }) !== undefined && + matsCollections.threshold !== undefined && + matsCollections.threshold.findOne({ + name: "threshold", + }) !== undefined + ) { + // get list of apps (variables in apps that also have thresholds) + apps = matsCollections.variable.findOne({ + name: "variable", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + for (aidx = 0; aidx < apps.length; aidx += 1) { + result[apps[aidx]] = matsCollections.variable.findOne({ + name: "variable", + }).optionsMap[apps[aidx]]; + if ( + typeof result[apps[aidx]] !== "string" && + !(result[apps[aidx]] instanceof String) + ) + result[apps[aidx]] = result[apps[aidx]].sumsDB; } - ); + } else { + result[matsCollections.Settings.findOne().Title] = + matsCollections.Databases.findOne({ + role: matsTypes.DatabaseRoles.SUMS_DATA, + status: "active", + }).database; + } + return result; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getApps`, - function (params, req, res, next) { - Picker.middleware(_getApps(params, req, res, next)); +// helper function for getting a metadata map from a MATS selector, keyed by app title and model display text +function getMapByAppAndModel(selector, mapType) { + let flatJSON = ""; + try { + let result; + if ( + matsCollections[selector] !== undefined && + matsCollections[selector].findOne({ name: selector }) !== undefined && + matsCollections[selector].findOne({ name: selector })[mapType] !== undefined + ) { + // get map of requested selector's metadata + result = matsCollections[selector].findOne({ + name: selector, + })[mapType]; + let newResult = {}; + if ( + mapType === "valuesMap" || + selector === "variable" || + selector === "statistic" + ) { + // valueMaps always need to be re-keyed by app (statistic and variable get their valuesMaps from optionsMaps) + newResult = mapMapToApps(result); + result = newResult; + } else if ( + matsCollections.database === undefined && + !( + matsCollections.variable !== undefined && + matsCollections.threshold !== undefined + ) + ) { + // key by app title if we're not already + const appTitle = matsCollections.Settings.findOne().Title; + newResult[appTitle] = result; + result = newResult; + } + } else { + result = {}; } - ); - - Picker.route("/getAppSumsDBs", function (params, req, res, next) { - Picker.middleware(_getAppSumsDBs(params, req, res, next)); - }); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log(`error retrieving metadata from ${selector}: `, e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getAppSumsDBs`, - function (params, req, res, next) { - Picker.middleware(_getAppSumsDBs(params, req, res, next)); +// helper function for getting a date metadata map from a MATS selector, keyed by app title and model display text +function getDateMapByAppAndModel() { + let flatJSON = ""; + try { + let result; + // the date map can be in a few places. we have to hunt for it. + if ( + matsCollections.database !== undefined && + matsCollections.database.findOne({ + name: "database", + }) !== undefined && + matsCollections.database.findOne({ + name: "database", + }).dates !== undefined + ) { + result = matsCollections.database.findOne({ + name: "database", + }).dates; + } else if ( + matsCollections.variable !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }) !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }).dates !== undefined + ) { + result = matsCollections.variable.findOne({ + name: "variable", + }).dates; + } else if ( + matsCollections["data-source"] !== undefined && + matsCollections["data-source"].findOne({ + name: "data-source", + }) !== undefined && + matsCollections["data-source"].findOne({ + name: "data-source", + }).dates !== undefined + ) { + result = matsCollections["data-source"].findOne({ + name: "data-source", + }).dates; + } else { + result = {}; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getAppSumsDBs`, - function (params, req, res, next) { - Picker.middleware(_getAppSumsDBs(params, req, res, next)); + if ( + matsCollections.database === undefined && + !( + matsCollections.variable !== undefined && + matsCollections.threshold !== undefined + ) + ) { + // key by app title if we're not already + const appTitle = matsCollections.Settings.findOne().Title; + const newResult = {}; + newResult[appTitle] = result; + result = newResult; } - ); - - Picker.route("/getModels", function (params, req, res, next) { - Picker.middleware(_getModels(params, req, res, next)); - }); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving datemap", e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getModels`, - function (params, req, res, next) { - Picker.middleware(_getModels(params, req, res, next)); +// helper function for getting a metadata map from a MATS selector, keyed by app title +function getMapByApp(selector) { + let flatJSON = ""; + try { + let result; + if ( + matsCollections[selector] !== undefined && + matsCollections[selector].findOne({ name: selector }) !== undefined + ) { + // get array of requested selector's metadata + result = matsCollections[selector].findOne({ + name: selector, + }).options; + if (!Array.isArray(result)) result = Object.keys(result); + } else if (selector === "statistic") { + result = ["ACC"]; + } else if (selector === "variable") { + result = [matsCollections.Settings.findOne().Title]; + } else { + result = []; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getModels`, - function (params, req, res, next) { - Picker.middleware(_getModels(params, req, res, next)); + // put results in a map keyed by app + let newResult; + if (result.length === 0) { + newResult = {}; + } else { + newResult = mapArrayToApps(result); } - ); - - Picker.route("/getRegions", function (params, req, res, next) { - Picker.middleware(_getRegions(params, req, res, next)); - }); + flatJSON = JSON.stringify(newResult); + } catch (e) { + console.log(`error retrieving metadata from ${selector}: `, e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getRegions`, - function (params, req, res, next) { - Picker.middleware(_getRegions(params, req, res, next)); +// helper function for populating the levels in a MATS selector +function getlevelsByApp() { + let flatJSON = ""; + try { + let result; + if ( + matsCollections.level !== undefined && + matsCollections.level.findOne({ name: "level" }) !== undefined + ) { + // we have levels already defined + result = matsCollections.level.findOne({ + name: "level", + }).options; + if (!Array.isArray(result)) result = Object.keys(result); + } else if ( + matsCollections.top !== undefined && + matsCollections.top.findOne({ name: "top" }) !== undefined + ) { + // use the MATS mandatory levels + result = _.range(100, 1050, 50); + if (!Array.isArray(result)) result = Object.keys(result); + } else { + result = []; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getRegions`, - function (params, req, res, next) { - Picker.middleware(_getRegions(params, req, res, next)); + let newResult; + if (result.length === 0) { + newResult = {}; + } else { + newResult = mapArrayToApps(result); } - ); - - Picker.route("/getRegionsValuesMap", function (params, req, res, next) { - Picker.middleware(_getRegionsValuesMap(params, req, res, next)); - }); + flatJSON = JSON.stringify(newResult); + } catch (e) { + console.log("error retrieving levels: ", e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getRegionsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getRegionsValuesMap(params, req, res, next)); +// private middleware for getApps route +const getApps = function (res) { + // this function returns an array of apps. + if (Meteor.isServer) { + let flatJSON = ""; + try { + const result = getListOfApps(); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving apps: ", e); + flatJSON = JSON.stringify({ + error: e, + }); } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getRegionsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getRegionsValuesMap(params, req, res, next)); +// private middleware for getAppSumsDBs route +const getAppSumsDBs = function (res) { + // this function returns map of apps and appRefs. + if (Meteor.isServer) { + let flatJSON = ""; + try { + const result = getListOfAppDBs(); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving apps: ", e); + flatJSON = JSON.stringify({ + error: e, + }); } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getStatistics", function (params, req, res, next) { - Picker.middleware(_getStatistics(params, req, res, next)); - }); +// private middleware for getModels route +const getModels = function (res) { + // this function returns a map of models keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("data-source", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getStatistics`, - function (params, req, res, next) { - Picker.middleware(_getStatistics(params, req, res, next)); +// private middleware for getRegions route +const getRegions = function (res) { + // this function returns a map of regions keyed by app title and model display text + if (Meteor.isServer) { + let flatJSON = getMapByAppAndModel("region", "optionsMap"); + if (flatJSON === "{}") { + flatJSON = getMapByAppAndModel("vgtyp", "optionsMap"); } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getStatistics`, - function (params, req, res, next) { - Picker.middleware(_getStatistics(params, req, res, next)); +// private middleware for getRegionsValuesMap route +const getRegionsValuesMap = function (res) { + // this function returns a map of regions values keyed by app title + if (Meteor.isServer) { + let flatJSON = getMapByAppAndModel("region", "valuesMap"); + if (flatJSON === "{}") { + flatJSON = getMapByAppAndModel("vgtyp", "valuesMap"); } - ); - - Picker.route("/getStatisticsValuesMap", function (params, req, res, next) { - Picker.middleware(_getStatisticsValuesMap(params, req, res, next)); - }); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getStatisticsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getStatisticsValuesMap(params, req, res, next)); - } - ); +// private middleware for getStatistics route +const getStatistics = function (res) { + // this function returns an map of statistics keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByApp("statistic"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getStatisticsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getStatisticsValuesMap(params, req, res, next)); - } - ); +// private middleware for getStatisticsValuesMap route +const getStatisticsValuesMap = function (res) { + // this function returns a map of statistic values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("statistic", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getVariables", function (params, req, res, next) { - Picker.middleware(_getVariables(params, req, res, next)); - }); +// private middleware for getVariables route +const getVariables = function (res) { + // this function returns an map of variables keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByApp("variable"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getVariables`, - function (params, req, res, next) { - Picker.middleware(_getVariables(params, req, res, next)); - } - ); +// private middleware for getVariablesValuesMap route +const getVariablesValuesMap = function (res) { + // this function returns a map of variable values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("variable", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getVariables`, - function (params, req, res, next) { - Picker.middleware(_getVariables(params, req, res, next)); - } - ); +// private middleware for getThresholds route +const getThresholds = function (res) { + // this function returns a map of thresholds keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("threshold", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getVariablesValuesMap", function (params, req, res, next) { - Picker.middleware(_getVariablesValuesMap(params, req, res, next)); - }); +// private middleware for getThresholdsValuesMap route +const getThresholdsValuesMap = function (res) { + // this function returns a map of threshold values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("threshold", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getVariablesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getVariablesValuesMap(params, req, res, next)); - } - ); +// private middleware for getScales route +const getScales = function (res) { + // this function returns a map of scales keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("scale", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getVariablesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getVariablesValuesMap(params, req, res, next)); - } - ); +// private middleware for getScalesValuesMap route +const getScalesValuesMap = function (res) { + // this function returns a map of scale values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("scale", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getThresholds", function (params, req, res, next) { - Picker.middleware(_getThresholds(params, req, res, next)); - }); +// private middleware for getTruth route +const getTruths = function (res) { + // this function returns a map of truths keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("truth", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getThresholds`, - function (params, req, res, next) { - Picker.middleware(_getThresholds(params, req, res, next)); - } - ); +// private middleware for getTruthValuesMap route +const getTruthsValuesMap = function (res) { + // this function returns a map of truth values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("truth", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholds`, - function (params, req, res, next) { - Picker.middleware(_getThresholds(params, req, res, next)); - } - ); +// private middleware for getFcstLengths route +const getFcstLengths = function (res) { + // this function returns a map of forecast lengths keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("forecast-length", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getThresholdsValuesMap", function (params, req, res, next) { - Picker.middleware(_getThresholdsValuesMap(params, req, res, next)); - }); +// private middleware for getFcstTypes route +const getFcstTypes = function (res) { + // this function returns a map of forecast types keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("forecast-type", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getThresholdsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getThresholdsValuesMap(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholdsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getThresholdsValuesMap(params, req, res, next)); - } - ); - - Picker.route("/getScales", function (params, req, res, next) { - Picker.middleware(_getScales(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getScales`, - function (params, req, res, next) { - Picker.middleware(_getScales(params, req, res, next)); - } - ); +// private middleware for getFcstTypesValuesMap route +const getFcstTypesValuesMap = function (res) { + // this function returns a map of forecast type values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("forecast-type", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getScales`, - function (params, req, res, next) { - Picker.middleware(_getScales(params, req, res, next)); - } - ); +// private middleware for getValidTimes route +const getValidTimes = function (res) { + // this function returns an map of valid times keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByApp("valid-time"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getScalesValuesMap", function (params, req, res, next) { - Picker.middleware(_getScalesValuesMap(params, req, res, next)); - }); +// private middleware for getValidTimes route +const getLevels = function (res) { + // this function returns an map of pressure levels keyed by app title + if (Meteor.isServer) { + const flatJSON = getlevelsByApp(); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getScalesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getScalesValuesMap(params, req, res, next)); - } - ); +// private middleware for getDates route +const getDates = function (res) { + // this function returns a map of dates keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getDateMapByAppAndModel(); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getScalesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getScalesValuesMap(params, req, res, next)); +// helper function for getCSV +const stringifyCurveData = function (stringify, dataArray, res) { + const thisDataArray = dataArray[0]; + stringify.stringify( + thisDataArray, + { + header: true, + }, + function (err, output) { + if (err) { + console.log("error in getCSV:", err); + res.write(`error,${err.toLocaleString()}`); + res.end(`

getCSV Error! ${err.toLocaleString()}

`); + return; + } + res.write(output); + if (dataArray.length > 1) { + const newDataArray = dataArray.slice(1); + stringifyCurveData(stringify, newDataArray, res); + } else { + res.end(); + } } ); +}; - Picker.route("/getTruths", function (params, req, res, next) { - Picker.middleware(_getTruths(params, req, res, next)); - }); +// private method for getting pagenated data +// a newPageIndex of -1000 means get all the data (used for export) +// a newPageIndex of -2000 means get just the last page +const getPagenatedData = function (rky, p, np) { + if (Meteor.isServer) { + const key = rky; + const myPageIndex = p; + const newPageIndex = np; + let rawReturn; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getTruths`, - function (params, req, res, next) { - Picker.middleware(_getTruths(params, req, res, next)); + try { + const result = matsCache.getResult(key); + rawReturn = result === undefined ? undefined : result.result; // getResult structure is {key:something, result:resultObject} + } catch (e) { + console.log("getPagenatedData: Error - ", e); + return undefined; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getTruths`, - function (params, req, res, next) { - Picker.middleware(_getTruths(params, req, res, next)); + const ret = + rawReturn === undefined ? undefined : JSON.parse(JSON.stringify(rawReturn)); + let start; + let end; + let direction = 1; + if (newPageIndex === -1000) { + // all the data + start = 0; + end = Number.MAX_VALUE; + } else if (newPageIndex === -2000) { + // just the last page + start = -2000; + direction = -1; + } else if (myPageIndex <= newPageIndex) { + // proceed forward + start = (newPageIndex - 1) * 100; + end = newPageIndex * 100; + } else { + // move back + direction = -1; + start = newPageIndex * 100; + end = (newPageIndex + 1) * 100; } - ); - Picker.route("/getTruthsValuesMap", function (params, req, res, next) { - Picker.middleware(_getTruthsValuesMap(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getTruthsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getTruthsValuesMap(params, req, res, next)); + let dsiStart; + let dsiEnd; + for (let csi = 0; csi < ret.data.length; csi += 1) { + if (ret.data[csi].x && ret.data[csi].x.length > 100) { + dsiStart = start; + dsiEnd = end; + if (dsiStart > ret.data[csi].x.length || dsiStart === -2000) { + // show the last page if we either requested it specifically or are trying to navigate past it + dsiStart = Math.floor(rawReturn.data[csi].x.length / 100) * 100; + dsiEnd = rawReturn.data[csi].x.length; + if (dsiEnd === dsiStart) { + // make sure the last page isn't empty -= 1if rawReturn.data[csi].data.length/100 produces a whole number, + // dsiStart and dsiEnd would be the same. This makes sure that the last full page is indeed the last page, without a phantom empty page afterwards + dsiStart = dsiEnd - 100; + } + } + if (dsiStart < 0) { + // show the first page if we are trying to navigate before it + dsiStart = 0; + dsiEnd = 100; + } + if (dsiEnd < dsiStart) { + // make sure that the end is after the start + dsiEnd = dsiStart + 100; + } + if (dsiEnd > ret.data[csi].x.length) { + // make sure we don't request past the end -= 1 if results are one page, this should convert the + // start and end from 0 and 100 to 0 and whatever the end is. + dsiEnd = ret.data[csi].x.length; + } + ret.data[csi].x = rawReturn.data[csi].x.slice(dsiStart, dsiEnd); + ret.data[csi].y = rawReturn.data[csi].y.slice(dsiStart, dsiEnd); + ret.data[csi].stats = rawReturn.data[csi].stats.slice(dsiStart, dsiEnd); + ret.data[csi].glob_stats = rawReturn.data[csi].glob_stats; + } } - ); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getTruthsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getTruthsValuesMap(params, req, res, next)); + if (direction === 1) { + ret.dsiRealPageIndex = Math.floor(dsiEnd / 100); + } else { + ret.dsiRealPageIndex = Math.floor(dsiStart / 100); } - ); + ret.dsiTextDirection = direction; + return ret; + } + return null; +}; - Picker.route("/getFcstLengths", function (params, req, res, next) { - Picker.middleware(_getFcstLengths(params, req, res, next)); - }); +// private method for getting pagenated results and flattening them in order to be appropriate for text display. +const getFlattenedResultData = function (rk, p, np) { + if (Meteor.isServer) { + try { + const r = rk; + const thisP = p; + const thisNP = np; + // get the pagenated data + const result = getPagenatedData(r, thisP, thisNP); + // find the type + const { plotTypes } = result.basis.plotParams; + const plotType = _.invert(plotTypes).true; + // extract data + let isCTC = false; + let isModeSingle = false; + let isModePairs = false; + let labelSuffix; + const returnData = {}; + const stats = {}; + let curveData = []; // array of maps + const { data } = result; + const { dsiRealPageIndex } = result; + const { dsiTextDirection } = result; + let firstBestFitIndex = -1; + let bestFitIndexes = {}; + switch (plotType) { + case matsTypes.PlotTypes.timeSeries: + case matsTypes.PlotTypes.profile: + case matsTypes.PlotTypes.dieoff: + case matsTypes.PlotTypes.threshold: + case matsTypes.PlotTypes.validtime: + case matsTypes.PlotTypes.dailyModelCycle: + case matsTypes.PlotTypes.gridscale: + case matsTypes.PlotTypes.yearToYear: + switch (plotType) { + case matsTypes.PlotTypes.timeSeries: + case matsTypes.PlotTypes.dailyModelCycle: + labelSuffix = " time"; + break; + case matsTypes.PlotTypes.profile: + labelSuffix = " level"; + break; + case matsTypes.PlotTypes.dieoff: + labelSuffix = " forecast lead time"; + break; + case matsTypes.PlotTypes.validtime: + labelSuffix = " hour of day"; + break; + case matsTypes.PlotTypes.threshold: + labelSuffix = " threshold"; + break; + case matsTypes.PlotTypes.gridscale: + labelSuffix = " grid scale"; + break; + case matsTypes.PlotTypes.yearToYear: + labelSuffix = " year"; + break; + default: + labelSuffix = "x-value"; + } + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + // for each curve + isCTC = + data[ci] !== undefined && + ((data[ci].stats !== undefined && + data[ci].stats[0] !== undefined && + data[ci].stats[0].hit !== undefined) || + (data[ci].hitTextOutput !== undefined && + data[ci].hitTextOutput.length > 0)); + isModePairs = + data[ci] !== undefined && + data[ci].stats !== undefined && + data[ci].stats[0] !== undefined && + data[ci].stats[0].avgInterest !== undefined; + isModeSingle = + data[ci] !== undefined && + data[ci].stats !== undefined && + data[ci].stats[0] !== undefined && + data[ci].stats[0].nForecast !== undefined; + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + stats.label = data[ci].label; + stats.mean = data[ci].glob_stats.dMean; + stats["standard deviation"] = data[ci].glob_stats.sd; + stats.n = data[ci].glob_stats.nGood; + if ( + plotType === matsTypes.PlotTypes.timeSeries || + plotType === matsTypes.PlotTypes.profile + ) { + stats["standard error"] = data[ci].glob_stats.stdeBetsy; + stats.lag1 = data[ci].glob_stats.lag1; + } + stats.minimum = data[ci].glob_stats.minVal; + stats.maximum = data[ci].glob_stats.maxVal; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].x.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + if (plotType === matsTypes.PlotTypes.profile) { + curveDataElement[data[ci].label + labelSuffix] = data[ci].y[cdi]; + } else { + curveDataElement[data[ci].label + labelSuffix] = data[ci].x[cdi]; + } + if (isCTC) { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.n = data[ci].stats[cdi].n; + curveDataElement.hit = data[ci].stats[cdi].hit; + curveDataElement.fa = data[ci].stats[cdi].fa; + curveDataElement.miss = data[ci].stats[cdi].miss; + curveDataElement.cn = data[ci].stats[cdi].cn; + } else if (isModeSingle) { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.nForecast = data[ci].stats[cdi].nForecast; + curveDataElement.nMatched = data[ci].stats[cdi].nMatched; + curveDataElement.nSimple = data[ci].stats[cdi].nSimple; + curveDataElement.nTotal = data[ci].stats[cdi].nTotal; + } else if (isModePairs) { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.n = data[ci].stats[cdi].n; + curveDataElement.avgInterest = data[ci].stats[cdi].avgInterest; + } else { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.mean = data[ci].stats[cdi].mean; + curveDataElement["std dev"] = data[ci].stats[cdi].sd; + if ( + plotType === matsTypes.PlotTypes.timeSeries || + plotType === matsTypes.PlotTypes.profile + ) { + curveDataElement["std error"] = data[ci].stats[cdi].stdeBetsy; + curveDataElement.lag1 = data[ci].stats[cdi].lag1; + } + curveDataElement.n = data[ci].stats[cdi].nGood; + } + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.reliability: + case matsTypes.PlotTypes.roc: + case matsTypes.PlotTypes.performanceDiagram: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if ( + reservedWords.indexOf(data[ci].label) === -1 && + !data[ci].label.includes(matsTypes.ReservedWords.noSkill) + ) { + stats.label = data[ci].label; + if (plotType === matsTypes.PlotTypes.reliability) { + stats["sample climo"] = data[ci].glob_stats.sample_climo; + } else if (plotType === matsTypes.PlotTypes.roc) { + stats.auc = data[ci].glob_stats.auc; + } + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].y.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + if (plotType === matsTypes.PlotTypes.reliability) { + curveDataElement[`${data[ci].label} probability bin`] = + data[ci].stats[cdi].prob_bin; + if (data[ci].stats[cdi].hit_rate) { + curveDataElement["hit rate"] = data[ci].stats[cdi].hit_rate; + } else { + curveDataElement["observed frequency"] = + data[ci].stats[cdi].obs_freq; + } + } else { + curveDataElement[`${data[ci].label} bin value`] = + data[ci].stats[cdi].bin_value; + curveDataElement["probability of detection"] = + data[ci].stats[cdi].pody; + if (plotType === matsTypes.PlotTypes.roc) { + curveDataElement["probability of false detection"] = + data[ci].stats[cdi].pofd; + } else { + curveDataElement["success ratio"] = data[ci].stats[cdi].fa; + } + curveDataElement.n = data[ci].stats[cdi].n; + } + if (data[ci].stats[cdi].obs_y) { + curveDataElement.oy = data[ci].stats[cdi].obs_y; + curveDataElement.on = data[ci].stats[cdi].obs_n; + } else { + curveDataElement.hitcount = data[ci].stats[cdi].hit_count; + curveDataElement.fcstcount = data[ci].stats[cdi].fcst_count; + } + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.gridscaleProb: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].y.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} probability bin`] = + data[ci].stats[cdi].bin_value; + curveDataElement["number of grid points"] = data[ci].stats[cdi].n_grid; + curveDataElement.n = data[ci].stats[cdi].n; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.simpleScatter: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getFcstLengths`, - function (params, req, res, next) { - Picker.middleware(_getFcstLengths(params, req, res, next)); - } - ); + for (let cdi = 0; cdi < data[ci].y.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} bin value`] = + data[ci].stats[cdi].bin_value; + curveDataElement["x-stat"] = data[ci].stats[cdi].xstat; + curveDataElement["y-stat"] = data[ci].stats[cdi].ystat; + curveDataElement.n = data[ci].stats[cdi].n; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.map: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + stats.label = data[0].label; + stats["total number of obs"] = data[0].stats.reduce(function (prev, curr) { + return prev + curr.nTimes; + }, 0); + stats["mean difference"] = matsDataUtils.average(data[0].queryVal); + stats["standard deviation"] = matsDataUtils.stdev(data[0].queryVal); + stats["minimum time"] = data[0].stats.reduce(function (prev, curr) { + return prev < curr.min_time ? prev : curr.min_time; + }); + stats["minimum time"] = moment + .utc(stats["minimum time"] * 1000) + .format("YYYY-MM-DD HH:mm"); + stats["maximum time"] = data[0].stats.reduce(function (prev, curr) { + return prev > curr.max_time ? prev : curr.max_time; + }); + stats["maximum time"] = moment + .utc(stats["maximum time"] * 1000) + .format("YYYY-MM-DD HH:mm"); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstLengths`, - function (params, req, res, next) { - Picker.middleware(_getFcstLengths(params, req, res, next)); - } - ); + returnData.stats[data[0].label] = stats; - Picker.route("/getFcstTypes", function (params, req, res, next) { - Picker.middleware(_getFcstTypes(params, req, res, next)); - }); + isCTC = + data[0] !== undefined && + data[0].stats !== undefined && + data[0].stats[0] !== undefined && + data[0].stats[0].hit !== undefined; + for (let si = 0; si < data[0].siteName.length; si += 1) { + const curveDataElement = {}; + curveDataElement["site name"] = data[0].siteName[si]; + curveDataElement["number of times"] = data[0].stats[si].nTimes; + if (isCTC) { + curveDataElement.stat = data[0].queryVal[si]; + curveDataElement.hit = data[0].stats[si].hit; + curveDataElement.fa = data[0].stats[si].fa; + curveDataElement.miss = data[0].stats[si].miss; + curveDataElement.cn = data[0].stats[si].cn; + } else { + curveDataElement["start date"] = moment + .utc(data[0].stats[si].min_time * 1000) + .format("YYYY-MM-DD HH:mm"); + curveDataElement["end date"] = moment + .utc(data[0].stats[si].max_time * 1000) + .format("YYYY-MM-DD HH:mm"); + curveDataElement.stat = data[0].queryVal[si]; + } + curveData.push(curveDataElement); + } + returnData.data[data[0].label] = curveData; + break; + case matsTypes.PlotTypes.histogram: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; + stats.mean = data[ci].glob_stats.glob_mean; + stats["standard deviation"] = data[ci].glob_stats.glob_sd; + stats.n = data[ci].glob_stats.glob_n; + stats.minimum = data[ci].glob_stats.glob_min; + stats.maximum = data[ci].glob_stats.glob_max; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].x.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} bin range`] = + data[ci].bin_stats[cdi].binLabel; + curveDataElement.n = data[ci].bin_stats[cdi].bin_n; + curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; + curveDataElement["bin lower bound"] = + data[ci].bin_stats[cdi].binLowBound; + curveDataElement["bin upper bound"] = + data[ci].bin_stats[cdi].binUpBound; + curveDataElement["bin mean"] = data[ci].bin_stats[cdi].bin_mean; + curveDataElement["bin std dev"] = data[ci].bin_stats[cdi].bin_sd; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.ensembleHistogram: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; + stats.mean = data[ci].glob_stats.dMean; + stats["standard deviation"] = data[ci].glob_stats.sd; + stats.n = data[ci].glob_stats.nGood; + stats.minimum = data[ci].glob_stats.minVal; + stats.maximum = data[ci].glob_stats.maxVal; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].x.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} bin`] = data[ci].x[cdi]; + curveDataElement.n = data[ci].bin_stats[cdi].bin_n; + curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.contour: + case matsTypes.PlotTypes.contourDiff: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + stats.label = data[0].label; + stats["total number of points"] = data[0].glob_stats.n; + stats["mean stat"] = data[0].glob_stats.mean; + stats["minimum time"] = moment + .utc(data[0].glob_stats.minDate * 1000) + .format("YYYY-MM-DD HH:mm"); + stats["maximum time"] = moment + .utc(data[0].glob_stats.maxDate * 1000) + .format("YYYY-MM-DD HH:mm"); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getFcstTypes`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypes(params, req, res, next)); - } - ); + returnData.stats[data[0].label] = stats; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypes`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypes(params, req, res, next)); + isCTC = + data[0] !== undefined && + data[0].hitTextOutput !== undefined && + data[0].hitTextOutput.length > 0; + for (let si = 0; si < data[0].xTextOutput.length; si += 1) { + const curveDataElement = {}; + curveDataElement.xVal = data[0].xTextOutput[si]; + curveDataElement.yVal = data[0].yTextOutput[si]; + curveDataElement.stat = data[0].zTextOutput[si]; + curveDataElement.N = data[0].nTextOutput[si]; + if (isCTC) { + curveDataElement.hit = data[0].hitTextOutput[si]; + curveDataElement.fa = data[0].faTextOutput[si]; + curveDataElement.miss = data[0].missTextOutput[si]; + curveDataElement.cn = data[0].cnTextOutput[si]; + } else { + curveDataElement["Start Date"] = moment + .utc(data[0].minDateTextOutput[si] * 1000) + .format("YYYY-MM-DD HH:mm"); + curveDataElement["End Date"] = moment + .utc(data[0].maxDateTextOutput[si] * 1000) + .format("YYYY-MM-DD HH:mm"); + } + curveData.push(curveDataElement); + } + returnData.data[data[0].label] = curveData; + break; + case matsTypes.PlotTypes.scatter2d: + firstBestFitIndex = -1; + bestFitIndexes = {}; + for (let ci = 0; ci < data.length; ci += 1) { + if (ci === firstBestFitIndex) { + break; // best fit curves are at the end so do not do further processing + } + curveData = data[ci]; + // look for a best fit curve - only have to look at curves with higher index than this one + for (let cbi = ci + 1; cbi < data.length; cbi += 1) { + if ( + data[cbi].label.indexOf(curveData.label) !== -1 && + data[cbi].label.indexOf("-best fit") !== -1 + ) { + bestFitIndexes[ci] = cbi; + if (firstBestFitIndex === -1) { + firstBestFitIndex = cbi; + } + break; + } + } + const curveTextData = []; + for (let cdi = 0; cdi < curveData.data.length; cdi += 1) { + const element = {}; + [element.xAxis] = curveData.data[cdi]; + [, element.yAxis] = curveData.data[cdi]; + if (bestFitIndexes[ci] === undefined) { + element["best fit"] = "none;"; + } else { + [, element["best fit"]] = data[bestFitIndexes[ci]].data[cdi]; + } + curveTextData.push(element); + } + returnData[curveData.label] = curveTextData; + } + break; + default: + return undefined; + } + returnData.dsiRealPageIndex = dsiRealPageIndex; + returnData.dsiTextDirection = dsiTextDirection; + return returnData; + } catch (error) { + throw new Meteor.Error( + `Error in getFlattenedResultData function: ${error.message}` + ); } - ); + } + return null; +}; - Picker.route("/getFcstTypesValuesMap", function (params, req, res, next) { - Picker.middleware(_getFcstTypesValuesMap(params, req, res, next)); - }); +// private middleware for getCSV route +const getCSV = function (params, res) { + if (Meteor.isServer) { + const stringify = require("csv-stringify"); + let csv = ""; + try { + const result = getFlattenedResultData(params.key, 0, -1000); + const statArray = Object.values(result.stats); + const dataArray = Object.values(result.data); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getFcstTypesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypesValuesMap(params, req, res, next)); + const fileName = `matsplot-${moment.utc().format("YYYYMMDD-HH.mm.ss")}.csv`; + res.setHeader("Content-disposition", `attachment; filename=${fileName}`); + res.setHeader("Content-Type", "attachment.ContentType"); + stringify.stringify( + statArray, + { + header: true, + }, + function (err, output) { + if (err) { + console.log("error in getCSV:", err); + res.write(`error,${err.toLocaleString()}`); + res.end(`

getCSV Error! ${err.toLocaleString()}

`); + return; + } + res.write(output); + stringifyCurveData(stringify, dataArray, res); + } + ); + } catch (e) { + console.log("error retrieving data: ", e); + csv = `error,${e.toLocaleString()}`; + res.setHeader("Content-disposition", "attachment; filename=matsplot.csv"); + res.setHeader("Content-Type", "attachment.ContentType"); + res.end(`

getCSV Error! ${csv}

`); } - ); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypesValuesMap(params, req, res, next)); +// private middleware for getJSON route +const getJSON = function (params, res) { + if (Meteor.isServer) { + let flatJSON = ""; + try { + const result = getPagenatedData(params.key, 0, -1000); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving data: ", e); + flatJSON = JSON.stringify({ + error: e, + }); + delete flatJSON.dsiRealPageIndex; + delete flatJSON.dsiTextDirection; } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getValidTimes", function (params, req, res, next) { - Picker.middleware(_getValidTimes(params, req, res, next)); - }); +// private define a middleware for refreshing the metadata +const refreshMetadataMWltData = function (res) { + if (Meteor.isServer) { + console.log("Server route asked to refresh metadata"); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getValidTimes`, - function (params, req, res, next) { - Picker.middleware(_getValidTimes(params, req, res, next)); + try { + console.log("GUI asked to refresh metadata"); + checkMetaDataRefresh(); + } catch (e) { + console.log(e); + res.end( + `` + + `

refreshMetadata Failed!

` + + `

${e.message}

` + + `` + ); } - ); + res.end("

refreshMetadata Done!

"); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getValidTimes`, - function (params, req, res, next) { - Picker.middleware(_getValidTimes(params, req, res, next)); +// private middleware for causing the scorecard to refresh its mongo collection for a given document +const refreshScorecard = function (params, res) { + if (Meteor.isServer) { + const docId = decodeURIComponent(params.docId); + // get userName, name, submitted, processedAt from id + // SC:anonymous -= 1submitted:20230322230435 -= 11block:0:02/19/2023_20_00_-_03/21/2023_20_00 + if (cbScorecardPool === undefined) { + throw new Meteor.Error("getScorecardData: No cbScorecardPool defined"); } - ); - - Picker.route("/getLevels", function (params, req, res, next) { - Picker.middleware(_getLevels(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getLevels`, - function (params, req, res, next) { - Picker.middleware(_getLevels(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getLevels`, - function (params, req, res, next) { - Picker.middleware(_getLevels(params, req, res, next)); - } - ); - - Picker.route("/getDates", function (params, req, res, next) { - Picker.middleware(_getDates(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getDates`, - function (params, req, res, next) { - Picker.middleware(_getDates(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getDates`, - function (params, req, res, next) { - Picker.middleware(_getDates(params, req, res, next)); - } - ); - - // create picker routes for refreshMetaData - Picker.route("/refreshMetadata", function (params, req, res, next) { - Picker.middleware(_refreshMetadataMWltData(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/refreshMetadata`, - function (params, req, res, next) { - Picker.middleware(_refreshMetadataMWltData(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/refreshMetadata`, - function (params, req, res, next) { - Picker.middleware(_refreshMetadataMWltData(params, req, res, next)); - } - ); - Picker.route("/refreshScorecard/:docId", function (params, req, res, next) { - Picker.middleware(_refreshScorecard(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/refreshScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_refreshScorecard(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/refreshScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_refreshScorecard(params, req, res, next)); - } - ); - - Picker.route("/setStatusScorecard/:docId", function (params, req, res, next) { - Picker.middleware(_setStatusScorecard(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/setStatusScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_setStatusScorecard(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/setStatusScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_setStatusScorecard(params, req, res, next)); - } - ); -} - -// private - used to see if the main page needs to update its selectors -const _checkMetaDataRefresh = async function () { - // This routine compares the current last modified time of the tables (MYSQL) or documents (Couchbase) - // used for curveParameter metadata with the last update time to determine if an update is necessary. - // We really only do this for Curveparams - /* - metaDataTableUpdates: - { - name: dataBaseName(MYSQL) or bucketName(couchbase), - (for couchbase tables are documents) - tables: [tableName1, tableName2 ..], - lastRefreshed : timestamp - } - */ - let refresh = false; - const tableUpdates = metaDataTableUpdates.find({}).fetch(); - const dbType = - matsCollections.Settings.findOne() !== undefined - ? matsCollections.Settings.findOne().dbType - : matsTypes.DbTypes.mysql; - for (let tui = 0; tui < tableUpdates.length; tui++) { - const id = tableUpdates[tui]._id; - const poolName = tableUpdates[tui].pool; - const dbName = tableUpdates[tui].name; - const tableNames = tableUpdates[tui].tables; - const { lastRefreshed } = tableUpdates[tui]; - let updatedEpoch = Number.MAX_VALUE; - for (let ti = 0; ti < tableNames.length; ti++) { - const tName = tableNames[ti]; - try { - if (Meteor.isServer) { - switch (dbType) { - case matsTypes.DbTypes.mysql: - var rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( - global[poolName], - `SELECT UNIX_TIMESTAMP(UPDATE_TIME)` + - ` FROM information_schema.tables` + - ` WHERE TABLE_SCHEMA = '${dbName}'` + - ` AND TABLE_NAME = '${tName}'` - ); - updatedEpoch = rows[0]["UNIX_TIMESTAMP(UPDATE_TIME)"]; - break; - case matsTypes.DbTypes.couchbase: - // the tName for couchbase is supposed to be the document id - var doc = await cbPool.getCB(tName); - updatedEpoch = doc.updated; - break; - default: - throw new Meteor.Error("resetApp: undefined DbType"); - } - } - // console.log("DB says metadata for table " + dbName + "." + tName + " was updated at " + updatedEpoch); - if ( - updatedEpoch === undefined || - updatedEpoch === null || - updatedEpoch === "NULL" || - updatedEpoch === Number.MAX_VALUE - ) { - // if time of last update isn't stored by the database (thanks, Aurora DB), refresh automatically - // console.log("_checkMetaDataRefresh - cannot find last update time for database: " + dbName + " and table: " + tName); - refresh = true; - // console.log("FORCED Refreshing the metadata for table because updatedEpoch is undefined" + dbName + "." + tName + " : updated at " + updatedEpoch); - break; + const statement = `SELECT sc.* + From + vxdata._default.SCORECARD sc + WHERE + sc.id='${docId}';`; + cbScorecardPool + .queryCB(statement) + .then((result) => { + // insert this result into the mongo Scorecard collection - createdAt is used for TTL + // created at gets updated each display even if it already existed. + // TTL is 24 hours + if (typeof result === "string") { + throw new Error(`Error from couchbase query - ${result}`); + } else if (result[0] === undefined) { + throw new Error("Error from couchbase query - document not found"); + } else { + matsCollections.Scorecard.upsert( + { + "scorecard.userName": result[0].userName, + "scorecard.name": result[0].name, + "scorecard.submitted": result[0].submitted, + "scorecard.processedAt": result[0].processedAt, + }, + { + $set: { + createdAt: new Date(), + scorecard: result[0], + }, + } + ); } - } catch (e) { - throw new Error( - `_checkMetaDataRefresh - error finding last update time for database: ${dbName} and table: ${tName}, ERROR:${e.message}` + res.end("

refreshScorecard Done!

"); + }) + .catch((err) => { + res.end( + `` + + `

refreshScorecard Failed!

` + + `

${err.message}

` + + `` ); - } - const lastRefreshedEpoch = moment.utc(lastRefreshed).valueOf() / 1000; - const updatedEpochMoment = moment.utc(updatedEpoch).valueOf(); - // console.log("Epoch of when this app last refreshed metadata for table " + dbName + "." + tName + " is " + lastRefreshedEpoch); - // console.log("Epoch of when the DB says table " + dbName + "." + tName + " was last updated is " + updatedEpochMoment); - if (lastRefreshedEpoch < updatedEpochMoment || updatedEpochMoment === 0) { - // Aurora DB sometimes returns a 0 for last updated. In that case, do refresh the metadata. - refresh = true; - // console.log("Refreshing the metadata in the app selectors because table " + dbName + "." + tName + " was updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " while the metadata was last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); - break; - } else { - // console.log("NOT Refreshing the metadata for table " + dbName + "." + tName + " : updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " : metadata last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); - } - } - if (refresh === true) { - // refresh the app metadata - // app specific routines - for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { - await global.appSpecificResetRoutines[ai](); - } - // remember that we updated ALL the metadata tables just now - metaDataTableUpdates.update( - { - _id: id, - }, - { - $set: { - lastRefreshed: moment().format(), - }, - } - ); - } + }); } - return true; }; -// private middleware for getting the status - think health check -const _status = function (params, req, res, next) { +const setStatusScorecard = function (params, req, res) { if (Meteor.isServer) { - const settings = matsCollections.Settings.findOne(); - res.end( - `
Running: version - ${settings.appVersion}
` + const docId = decodeURIComponent(params.docId); + let body = ""; + req.on( + "data", + Meteor.bindEnvironment(function (data) { + body += data; + }) + ); + + req.on( + "end", + Meteor.bindEnvironment(function () { + // console.log(body); + try { + const doc = JSON.parse(body); + const docStatus = doc.status; + const found = matsCollections.Scorecard.find({ id: docId }).fetch(); + if (found.length === 0) { + throw new Error("Error from scorecard lookup - document not found"); + } + matsCollections.Scorecard.upsert( + { + id: docId, + }, + { + $set: { + docStatus, + }, + } + ); + // set error if there is one somehow. (use the session?) + res.end("

setScorecardStatus Done!

"); + } catch (err) { + res.statusCode = 400; + res.end( + `` + + `

setScorecardStatus Failed!

` + + `

${err.message}

` + + `` + ); + } + }) ); } }; -// private middleware for clearing the cache -const _clearCache = function (params, req, res, next) { +// private save the result from the query into mongo and downsample if that result's size is greater than 1.2Mb +const saveResultData = function (result) { if (Meteor.isServer) { - matsCache.clear(); - res.end("

clearCache Done!

"); + const storedResult = result; + const sizeof = require("object-sizeof"); + const hash = require("object-hash"); + const key = hash(storedResult.basis.plotParams); + const threshold = 1200000; + let ret = {}; + try { + const dSize = sizeof(storedResult.data); + // console.log("storedResult.basis.data size is ", dSize); + // TimeSeries and DailyModelCycle are the only plot types that require downSampling + if ( + dSize > threshold && + (storedResult.basis.plotParams.plotTypes.TimeSeries || + storedResult.basis.plotParams.plotTypes.DailyModelCycle) + ) { + // greater than threshold need to downsample + // downsample and save it in DownSampleResult + console.log("DownSampling"); + const downsampler = require("downsample-lttb"); + let totalPoints = 0; + for (let di = 0; di < storedResult.data.length; di += 1) { + totalPoints += storedResult.data[di].x_epoch.length; + } + const allowedNumberOfPoints = (threshold / dSize) * totalPoints; + const downSampleResult = + storedResult === undefined + ? undefined + : JSON.parse(JSON.stringify(storedResult)); + for (let ci = 0; ci < storedResult.data.length; ci += 1) { + const dsData = {}; + const xyDataset = storedResult.data[ci].x_epoch.map(function (d, index) { + return [ + storedResult.data[ci].x_epoch[index], + storedResult.data[ci].y[index], + ]; + }); + const ratioTotalPoints = xyDataset.length / totalPoints; + const myAllowedPoints = Math.round(ratioTotalPoints * allowedNumberOfPoints); + // downsample the array + let downsampledSeries; + if (myAllowedPoints < xyDataset.length && xyDataset.length > 2) { + downsampledSeries = downsampler.processData(xyDataset, myAllowedPoints); + // replace the y attributes (tooltips etc.) with the y attributes from the nearest x + let originalIndex = 0; + // skip through the original dataset capturing each downSampled data point + const arrayKeys = []; + const nonArrayKeys = []; + const keys = Object.keys(storedResult.data[ci]); + for (let ki = 0; ki < keys.length; ki += 1) { + if (keys[ki] !== "x_epoch") { + if (Array.isArray(storedResult.data[ci][keys[ki]])) { + arrayKeys.push(keys[ki]); + dsData[keys[ki]] = []; + } else { + nonArrayKeys.push(keys[ki]); + } + } + } + // We only ever downsample series plots - never profiles and series plots only ever have error_y arrays. + // This is a little hacky but what is happening is we putting error_y.array on the arrayKeys list so that it gets its + // downsampled equivalent values. + for (let ki = 0; ki < nonArrayKeys.length; ki += 1) { + dsData[nonArrayKeys[ki]] = JSON.parse( + JSON.stringify(storedResult.data[ci][nonArrayKeys[ki]]) + ); + } + // remove the original error_y array data. + dsData.error_y.array = []; + for (let dsi = 0; dsi < downsampledSeries.length; dsi += 1) { + while ( + originalIndex < storedResult.data[ci].x_epoch.length && + storedResult.data[ci].x_epoch[originalIndex] < downsampledSeries[dsi][0] + ) { + originalIndex += 1; + } + // capture the stuff related to this downSampled data point (downSampled data points are always a subset of original data points) + for (let ki = 0; ki < arrayKeys.length; ki += 1) { + dsData[arrayKeys[ki]][dsi] = + storedResult.data[ci][arrayKeys[ki]][originalIndex]; + } + dsData.error_y.array[dsi] = + storedResult.data[ci].error_y.array[originalIndex]; + } + // add downsampled annotation to curve options + downSampleResult[ci] = dsData; + downSampleResult[ci].annotation += " **DOWNSAMPLED**"; + } else { + downSampleResult[ci] = storedResult.data[ci]; + } + downSampleResult.data[ci] = downSampleResult[ci]; + } + DownSampleResults.rawCollection().insert({ + createdAt: new Date(), + key, + result: downSampleResult, + }); // createdAt ensures expiration set in mats-collections + ret = { + key, + result: downSampleResult, + }; + } else { + ret = { + key, + result: storedResult, + }; + } + // save original dataset in the matsCache + if ( + storedResult.basis.plotParams.plotTypes.TimeSeries || + storedResult.basis.plotParams.plotTypes.DailyModelCycle + ) { + for (let ci = 0; ci < storedResult.data.length; ci += 1) { + delete storedResult.data[ci].x_epoch; // we only needed this as an index for downsampling + } + } + matsCache.storeResult(key, { + key, + result: storedResult, + }); // lifespan is handled by lowDb (internally) in matscache + } catch (error) { + if (error.toLocaleString().indexOf("larger than the maximum size") !== -1) { + throw new Meteor.Error(": Requesting too much data... try averaging"); + } + } + return ret; } + return null; }; -// private middleware for dropping a distinct instance (a single run) of a scorecard -const _dropScorecardInstance = async function ( - userName, - name, - submittedTime, - processedAt -) { +// Utility method for writing out the meteor.settings file +const writeSettings = function (settings, appName) { + const fs = require("fs"); + let settingsPath = process.env.METEOR_SETTINGS_DIR; + if (!settingsPath) { + console.log( + "environment var METEOR_SETTINGS_DIR is undefined: setting it to /usr/app/settings" + ); + settingsPath = "/usr/app/settings"; + } + if (!fs.existsSync(settingsPath)) { + fs.mkdirSync(settingsPath, { + recursive: true, + }); + } + let appSettings = {}; + let newSettings = {}; + try { + const appSettingsData = fs.readFileSync(`${settingsPath}/${appName}/settings.json`); + appSettings = JSON.parse(appSettingsData); + } catch (e) { + appSettings = { + private: {}, + public: {}, + }; + } + newSettings = settings; + // Merge settings into appSettings + newSettings.private = { + ...appSettings.private, + ...settings.private, + }; + newSettings.public = { + ...appSettings.public, + ...settings.public, + }; + // write the settings file + const jsonSettings = JSON.stringify(newSettings, null, 2); + // console.log (jsonSettings); + fs.writeFileSync(`${settingsPath}/${appName}/settings.json`, jsonSettings, { + encoding: "utf8", + flag: "w", + }); +}; +// return the scorecard for the provided selectors +const getThisScorecardData = async function (userName, name, submitted, processedAt) { try { if (cbScorecardPool === undefined) { - throw new Meteor.Error("_dropScorecardInstance: No cbScorecardPool defined"); + throw new Meteor.Error("getThisScorecardData: No cbScorecardPool defined"); } - const statement = `DELETE + const statement = `SELECT sc.* From vxdata._default.SCORECARD sc WHERE @@ -745,3095 +1715,2236 @@ const _dropScorecardInstance = async function ( AND sc.userName='${userName}' AND sc.name='${name}' AND sc.processedAt=${processedAt} - AND sc.submitted=${submittedTime};`; - const result = await cbScorecardPool.queryCB(statement); - // delete this result from the mongo Scorecard collection + AND sc.submitted=${submitted};`; + const result = await cbScorecardPool.queryCBWithConsistency(statement); + if (typeof result === "string" && result.indexOf("ERROR")) { + throw new Meteor.Error(result); + } + // insert this result into the mongo Scorecard collection - createdAt is used for TTL + // created at gets updated each display even if it already existed. + // TTL is 24 hours + matsCollections.Scorecard.upsert( + { + "scorecard.userName": result[0].userName, + "scorecard.name": result[0].name, + "scorecard.submitted": result[0].submitted, + "scorecard.processedAt": result[0].processedAt, + }, + { + $set: { + createdAt: new Date(), + scorecard: result[0], + }, + } + ); + const docID = matsCollections.Scorecard.findOne( + { + "scorecard.userName": result[0].userName, + "scorecard.name": result[0].name, + "scorecard.submitted": result[0].submitted, + "scorecard.processedAt": result[0].processedAt, + }, + { _id: 1 } + )._id; + // no need to return the whole thing, just the identifying fields + // and the ID. The app will find the whole thing in the mongo collection. + return { scorecard: result[0], docID }; } catch (err) { - console.log(`_dropScorecardInstance error : ${err.message}`); + console.log(`getThisScorecardData error : ${err.message}`); return { error: err.message, }; } }; -// helper function to map a results array to specific apps -function _mapArrayToApps(result) { - // put results in a map keyed by app - const newResult = {}; - const apps = _getListOfApps(); - for (let aidx = 0; aidx < apps.length; aidx++) { - if (result[aidx] === apps[aidx]) { - newResult[apps[aidx]] = [result[aidx]]; - } else { - newResult[apps[aidx]] = result; +// return the scorecard status information from the couchbase database +const getThisScorecardInfo = async function () { + try { + if (cbScorecardPool === undefined) { + throw new Meteor.Error("getThisScorecardInfo: No cbScorecardPool defined"); } - } - return newResult; -} -// helper function to map a results map to specific apps -function _mapMapToApps(result) { - // put results in a map keyed by app - let newResult = {}; - const apps = _getListOfApps(); - const resultKeys = Object.keys(result); - if (!matsDataUtils.arraysEqual(apps.sort(), resultKeys.sort())) { - if (resultKeys.includes("Predefined region")) result = result["Predefined region"]; - for (let aidx = 0; aidx < apps.length; aidx++) { - newResult[apps[aidx]] = result; - } - } else { - newResult = result; + const statement = `SELECT + sc.id, + sc.userName, + sc.name, + sc.status, + sc.processedAt as processedAt, + sc.submitted, + sc.dateRange + From + vxdata._default.SCORECARD sc + WHERE + sc.type='SC';`; + const result = await cbScorecardPool.queryCBWithConsistency(statement); + const scMap = {}; + result.forEach(function (elem) { + if (!Object.keys(scMap).includes(elem.userName)) { + scMap[elem.userName] = {}; + } + const userElem = scMap[elem.userName]; + if (!Object.keys(userElem).includes(elem.name)) { + userElem[elem.name] = {}; + } + const nameElem = userElem[elem.name]; + if (!Object.keys(nameElem).includes(elem.submited)) { + nameElem[elem.submitted] = {}; + } + const submittedElem = nameElem[elem.submitted]; + submittedElem[elem.processedAt] = { + id: elem.id, + status: elem.status, + submitted: elem.submitted, + }; + }); + return scMap; + } catch (err) { + console.log(`getThisScorecardInfo error : ${err.message}`); + return { + error: err.message, + }; } - return newResult; -} +}; -// helper function for returning an array of database-distinct apps contained within a larger MATS app -function _getListOfApps() { - let apps; - if ( - matsCollections.database !== undefined && - matsCollections.database.findOne({ name: "database" }) !== undefined - ) { - // get list of databases (one per app) - apps = matsCollections.database.findOne({ - name: "database", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - } else if ( - matsCollections.variable !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }) !== undefined && - matsCollections.threshold !== undefined && - matsCollections.threshold.findOne({ - name: "threshold", - }) !== undefined - ) { - // get list of apps (variables in apps that also have thresholds) - apps = matsCollections.variable.findOne({ - name: "variable", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - } else { - apps = [matsCollections.Settings.findOne().Title]; - } - return apps; -} - -// helper function for returning a map of database-distinct apps contained within a larger MATS app and their DBs -function _getListOfAppDBs() { - let apps; - const result = {}; - let aidx; - if ( - matsCollections.database !== undefined && - matsCollections.database.findOne({ name: "database" }) !== undefined - ) { - // get list of databases (one per app) - apps = matsCollections.database.findOne({ - name: "database", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - for (aidx = 0; aidx < apps.length; aidx++) { - result[apps[aidx]] = matsCollections.database.findOne({ - name: "database", - }).optionsMap[apps[aidx]].sumsDB; - } - } else if ( - matsCollections.variable !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }) !== undefined && - matsCollections.threshold !== undefined && - matsCollections.threshold.findOne({ - name: "threshold", - }) !== undefined - ) { - // get list of apps (variables in apps that also have thresholds) - apps = matsCollections.variable.findOne({ - name: "variable", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - for (aidx = 0; aidx < apps.length; aidx++) { - result[apps[aidx]] = matsCollections.variable.findOne({ - name: "variable", - }).optionsMap[apps[aidx]]; - if ( - typeof result[apps[aidx]] !== "string" && - !(result[apps[aidx]] instanceof String) - ) - result[apps[aidx]] = result[apps[aidx]].sumsDB; - } - } else { - result[matsCollections.Settings.findOne().Title] = - matsCollections.Databases.findOne({ - role: matsTypes.DatabaseRoles.SUMS_DATA, - status: "active", - }).database; - } - return result; -} - -// helper function for getting a metadata map from a MATS selector, keyed by app title and model display text -function _getMapByAppAndModel(selector, mapType) { - let flatJSON = ""; - try { - let result; - if ( - matsCollections[selector] !== undefined && - matsCollections[selector].findOne({ name: selector }) !== undefined && - matsCollections[selector].findOne({ name: selector })[mapType] !== undefined - ) { - // get map of requested selector's metadata - result = matsCollections[selector].findOne({ - name: selector, - })[mapType]; - let newResult = {}; - if ( - mapType === "valuesMap" || - selector === "variable" || - selector === "statistic" - ) { - // valueMaps always need to be re-keyed by app (statistic and variable get their valuesMaps from optionsMaps) - newResult = _mapMapToApps(result); - result = newResult; - } else if ( - matsCollections.database === undefined && - !( - matsCollections.variable !== undefined && - matsCollections.threshold !== undefined - ) - ) { - // key by app title if we're not already - const appTitle = matsCollections.Settings.findOne().Title; - newResult[appTitle] = result; - result = newResult; - } - } else { - result = {}; - } - flatJSON = JSON.stringify(result); - } catch (e) { - console.log(`error retrieving metadata from ${selector}: `, e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// helper function for getting a date metadata map from a MATS selector, keyed by app title and model display text -function _getDateMapByAppAndModel() { - let flatJSON = ""; - try { - let result; - // the date map can be in a few places. we have to hunt for it. - if ( - matsCollections.database !== undefined && - matsCollections.database.findOne({ - name: "database", - }) !== undefined && - matsCollections.database.findOne({ - name: "database", - }).dates !== undefined - ) { - result = matsCollections.database.findOne({ - name: "database", - }).dates; - } else if ( - matsCollections.variable !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }) !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }).dates !== undefined - ) { - result = matsCollections.variable.findOne({ - name: "variable", - }).dates; - } else if ( - matsCollections["data-source"] !== undefined && - matsCollections["data-source"].findOne({ - name: "data-source", - }) !== undefined && - matsCollections["data-source"].findOne({ - name: "data-source", - }).dates !== undefined - ) { - result = matsCollections["data-source"].findOne({ - name: "data-source", - }).dates; - } else { - result = {}; - } - if ( - matsCollections.database === undefined && - !( - matsCollections.variable !== undefined && - matsCollections.threshold !== undefined - ) - ) { - // key by app title if we're not already - const appTitle = matsCollections.Settings.findOne().Title; - const newResult = {}; - newResult[appTitle] = result; - result = newResult; - } - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving datemap", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// helper function for getting a metadata map from a MATS selector, keyed by app title -function _getMapByApp(selector) { - let flatJSON = ""; - try { - let result; - if ( - matsCollections[selector] !== undefined && - matsCollections[selector].findOne({ name: selector }) !== undefined - ) { - // get array of requested selector's metadata - result = matsCollections[selector].findOne({ - name: selector, - }).options; - if (!Array.isArray(result)) result = Object.keys(result); - } else if (selector === "statistic") { - result = ["ACC"]; - } else if (selector === "variable") { - result = [matsCollections.Settings.findOne().Title]; - } else { - result = []; - } - // put results in a map keyed by app - let newResult; - if (result.length === 0) { - newResult = {}; - } else { - newResult = _mapArrayToApps(result); - } - flatJSON = JSON.stringify(newResult); - } catch (e) { - console.log(`error retrieving metadata from ${selector}: `, e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// helper function for populating the levels in a MATS selector -function _getlevelsByApp() { - let flatJSON = ""; - try { - let result; - if ( - matsCollections.level !== undefined && - matsCollections.level.findOne({ name: "level" }) !== undefined - ) { - // we have levels already defined - result = matsCollections.level.findOne({ - name: "level", - }).options; - if (!Array.isArray(result)) result = Object.keys(result); - } else if ( - matsCollections.top !== undefined && - matsCollections.top.findOne({ name: "top" }) !== undefined - ) { - // use the MATS mandatory levels - result = _.range(100, 1050, 50); - if (!Array.isArray(result)) result = Object.keys(result); - } else { - result = []; - } - let newResult; - if (result.length === 0) { - newResult = {}; - } else { - newResult = _mapArrayToApps(result); - } - flatJSON = JSON.stringify(newResult); - } catch (e) { - console.log("error retrieving levels: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// private middleware for _getApps route -const _getApps = function (params, req, res, next) { - // this function returns an array of apps. - if (Meteor.isServer) { - let flatJSON = ""; - try { - const result = _getListOfApps(); - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving apps: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getAppSumsDBs route -const _getAppSumsDBs = function (params, req, res, next) { - // this function returns map of apps and appRefs. - if (Meteor.isServer) { - let flatJSON = ""; - try { - const result = _getListOfAppDBs(); - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving apps: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getModels route -const _getModels = function (params, req, res, next) { - // this function returns a map of models keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("data-source", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getRegions route -const _getRegions = function (params, req, res, next) { - // this function returns a map of regions keyed by app title and model display text - if (Meteor.isServer) { - let flatJSON = _getMapByAppAndModel("region", "optionsMap"); - if (flatJSON === "{}") { - flatJSON = _getMapByAppAndModel("vgtyp", "optionsMap"); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getRegionsValuesMap route -const _getRegionsValuesMap = function (params, req, res, next) { - // this function returns a map of regions values keyed by app title - if (Meteor.isServer) { - let flatJSON = _getMapByAppAndModel("region", "valuesMap"); - if (flatJSON === "{}") { - flatJSON = _getMapByAppAndModel("vgtyp", "valuesMap"); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getStatistics route -const _getStatistics = function (params, req, res, next) { - // this function returns an map of statistics keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByApp("statistic"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getStatisticsValuesMap route -const _getStatisticsValuesMap = function (params, req, res, next) { - // this function returns a map of statistic values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("statistic", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getVariables route -const _getVariables = function (params, req, res, next) { - // this function returns an map of variables keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByApp("variable"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getVariablesValuesMap route -const _getVariablesValuesMap = function (params, req, res, next) { - // this function returns a map of variable values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("variable", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getThresholds route -const _getThresholds = function (params, req, res, next) { - // this function returns a map of thresholds keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("threshold", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getThresholdsValuesMap route -const _getThresholdsValuesMap = function (params, req, res, next) { - // this function returns a map of threshold values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("threshold", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getScales route -const _getScales = function (params, req, res, next) { - // this function returns a map of scales keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("scale", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getScalesValuesMap route -const _getScalesValuesMap = function (params, req, res, next) { - // this function returns a map of scale values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("scale", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getTruth route -const _getTruths = function (params, req, res, next) { - // this function returns a map of truths keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("truth", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getTruthValuesMap route -const _getTruthsValuesMap = function (params, req, res, next) { - // this function returns a map of truth values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("truth", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getFcstLengths route -const _getFcstLengths = function (params, req, res, next) { - // this function returns a map of forecast lengths keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("forecast-length", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getFcstTypes route -const _getFcstTypes = function (params, req, res, next) { - // this function returns a map of forecast types keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("forecast-type", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getFcstTypesValuesMap route -const _getFcstTypesValuesMap = function (params, req, res, next) { - // this function returns a map of forecast type values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("forecast-type", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getValidTimes route -const _getValidTimes = function (params, req, res, next) { - // this function returns an map of valid times keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByApp("valid-time"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getValidTimes route -const _getLevels = function (params, req, res, next) { - // this function returns an map of pressure levels keyed by app title - if (Meteor.isServer) { - const flatJSON = _getlevelsByApp(); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getDates route -const _getDates = function (params, req, res, next) { - // this function returns a map of dates keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getDateMapByAppAndModel(); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// helper function for _getCSV -const stringifyCurveData = function (stringify, dataArray, res) { - const thisDataArray = dataArray[0]; - stringify.stringify( - thisDataArray, - { - header: true, - }, - function (err, output) { - if (err) { - console.log("error in _getCSV:", err); - res.write(`error,${err.toLocaleString()}`); - res.end(`

_getCSV Error! ${err.toLocaleString()}

`); - return; - } - res.write(output); - if (dataArray.length > 1) { - const newDataArray = dataArray.slice(1); - stringifyCurveData(stringify, newDataArray, res); - } else { - res.end(); - } - } - ); -}; - -// private middleware for _getCSV route -const _getCSV = function (params, req, res, next) { - if (Meteor.isServer) { - const stringify = require("csv-stringify"); - let csv = ""; - try { - const result = _getFlattenedResultData(params.key, 0, -1000); - const statArray = Object.values(result.stats); - const dataArray = Object.values(result.data); - - const fileName = `matsplot-${moment.utc().format("YYYYMMDD-HH.mm.ss")}.csv`; - res.setHeader("Content-disposition", `attachment; filename=${fileName}`); - res.setHeader("Content-Type", "attachment.ContentType"); - stringify.stringify( - statArray, - { - header: true, - }, - function (err, output) { - if (err) { - console.log("error in _getCSV:", err); - res.write(`error,${err.toLocaleString()}`); - res.end(`

_getCSV Error! ${err.toLocaleString()}

`); - return; - } - res.write(output); - stringifyCurveData(stringify, dataArray, res); - } +const getThesePlotParamsFromScorecardInstance = async function ( + userName, + name, + submitted, + processedAt +) { + try { + if (cbScorecardPool === undefined) { + throw new Meteor.Error( + "getThesePlotParamsFromScorecardInstance: No cbScorecardPool defined" ); - } catch (e) { - console.log("error retrieving data: ", e); - csv = `error,${e.toLocaleString()}`; - res.setHeader("Content-disposition", "attachment; filename=matsplot.csv"); - res.setHeader("Content-Type", "attachment.ContentType"); - res.end(`

_getCSV Error! ${csv}

`); } - } -}; - -// private middleware for _getJSON route -const _getJSON = function (params, req, res, next) { - if (Meteor.isServer) { - let flatJSON = ""; - try { - const result = _getPagenatedData(params.key, 0, -1000); - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving data: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - delete flatJSON.dsiRealPageIndex; - delete flatJSON.dsiTextDirection; + const statement = `SELECT sc.plotParams + From + vxdata._default.SCORECARD sc + WHERE + sc.type='SC' + AND sc.userName='${userName}' + AND sc.name='${name}' + AND sc.processedAt=${processedAt} + AND sc.submitted=${submitted};`; + const result = await cbScorecardPool.queryCBWithConsistency(statement); + if (typeof result === "string" && result.indexOf("ERROR")) { + throw new Meteor.Error(result); } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); + return result[0]; + } catch (err) { + console.log(`getThesePlotParamsFromScorecardInstance error : ${err.message}`); + return { + error: err.message, + }; } }; -// private method for getting pagenated results and flattening them in order to be appropriate for text display. -const _getFlattenedResultData = function (rk, p, np) { - if (Meteor.isServer) { - let resp; - try { - const r = rk; - var p = p; - var np = np; - // get the pagenated data - const result = _getPagenatedData(r, p, np); - // find the type - const { plotTypes } = result.basis.plotParams; - const plotType = _.invert(plotTypes).true; - // extract data - let isCTC = false; - let isModeSingle = false; - let isModePairs = false; - const { data } = result; - const { dsiRealPageIndex } = result; - const { dsiTextDirection } = result; - switch (plotType) { - case matsTypes.PlotTypes.timeSeries: - case matsTypes.PlotTypes.profile: - case matsTypes.PlotTypes.dieoff: - case matsTypes.PlotTypes.threshold: - case matsTypes.PlotTypes.validtime: - case matsTypes.PlotTypes.dailyModelCycle: - case matsTypes.PlotTypes.gridscale: - case matsTypes.PlotTypes.yearToYear: - var labelSuffix; - switch (plotType) { - case matsTypes.PlotTypes.timeSeries: - case matsTypes.PlotTypes.dailyModelCycle: - labelSuffix = " time"; - break; - case matsTypes.PlotTypes.profile: - labelSuffix = " level"; - break; - case matsTypes.PlotTypes.dieoff: - labelSuffix = " forecast lead time"; - break; - case matsTypes.PlotTypes.validtime: - labelSuffix = " hour of day"; - break; - case matsTypes.PlotTypes.threshold: - labelSuffix = " threshold"; - break; - case matsTypes.PlotTypes.gridscale: - labelSuffix = " grid scale"; - break; - case matsTypes.PlotTypes.yearToYear: - labelSuffix = " year"; - break; - } - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - isCTC = - data[ci] !== undefined && - ((data[ci].stats !== undefined && - data[ci].stats[0] !== undefined && - data[ci].stats[0].hit !== undefined) || - (data[ci].hitTextOutput !== undefined && - data[ci].hitTextOutput.length > 0)); - isModePairs = - data[ci] !== undefined && - data[ci].stats !== undefined && - data[ci].stats[0] !== undefined && - data[ci].stats[0].avgInterest !== undefined; - isModeSingle = - data[ci] !== undefined && - data[ci].stats !== undefined && - data[ci].stats[0] !== undefined && - data[ci].stats[0].nForecast !== undefined; - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.dMean; - stats["standard deviation"] = data[ci].glob_stats.sd; - stats.n = data[ci].glob_stats.nGood; - if ( - plotType === matsTypes.PlotTypes.timeSeries || - plotType === matsTypes.PlotTypes.profile - ) { - stats["standard error"] = data[ci].glob_stats.stdeBetsy; - stats.lag1 = data[ci].glob_stats.lag1; - } - stats.minimum = data[ci].glob_stats.minVal; - stats.maximum = data[ci].glob_stats.maxVal; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].x.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - if (plotType === matsTypes.PlotTypes.profile) { - curveDataElement[data[ci].label + labelSuffix] = data[ci].y[cdi]; - } else { - curveDataElement[data[ci].label + labelSuffix] = data[ci].x[cdi]; - } - if (isCTC) { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.n = data[ci].stats[cdi].n; - curveDataElement.hit = data[ci].stats[cdi].hit; - curveDataElement.fa = data[ci].stats[cdi].fa; - curveDataElement.miss = data[ci].stats[cdi].miss; - curveDataElement.cn = data[ci].stats[cdi].cn; - } else if (isModeSingle) { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.nForecast = data[ci].stats[cdi].nForecast; - curveDataElement.nMatched = data[ci].stats[cdi].nMatched; - curveDataElement.nSimple = data[ci].stats[cdi].nSimple; - curveDataElement.nTotal = data[ci].stats[cdi].nTotal; - } else if (isModePairs) { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.n = data[ci].stats[cdi].n; - curveDataElement.avgInterest = data[ci].stats[cdi].avgInterest; - } else { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.mean = data[ci].stats[cdi].mean; - curveDataElement["std dev"] = data[ci].stats[cdi].sd; - if ( - plotType === matsTypes.PlotTypes.timeSeries || - plotType === matsTypes.PlotTypes.profile - ) { - curveDataElement["std error"] = data[ci].stats[cdi].stdeBetsy; - curveDataElement.lag1 = data[ci].stats[cdi].lag1; - } - curveDataElement.n = data[ci].stats[cdi].nGood; - } - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.reliability: - case matsTypes.PlotTypes.roc: - case matsTypes.PlotTypes.performanceDiagram: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if ( - reservedWords.indexOf(data[ci].label) >= 0 || - data[ci].label.includes(matsTypes.ReservedWords.noSkill) - ) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - if (plotType === matsTypes.PlotTypes.reliability) { - stats["sample climo"] = data[ci].glob_stats.sample_climo; - } else if (plotType === matsTypes.PlotTypes.roc) { - stats.auc = data[ci].glob_stats.auc; - } - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].y.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - if (plotType === matsTypes.PlotTypes.reliability) { - curveDataElement[`${data[ci].label} probability bin`] = - data[ci].stats[cdi].prob_bin; - if (data[ci].stats[cdi].hit_rate) { - curveDataElement["hit rate"] = data[ci].stats[cdi].hit_rate; - } else { - curveDataElement["observed frequency"] = data[ci].stats[cdi].obs_freq; - } - } else { - curveDataElement[`${data[ci].label} bin value`] = - data[ci].stats[cdi].bin_value; - curveDataElement["probability of detection"] = data[ci].stats[cdi].pody; - if (plotType === matsTypes.PlotTypes.roc) { - curveDataElement["probability of false detection"] = - data[ci].stats[cdi].pofd; - } else { - curveDataElement["success ratio"] = data[ci].stats[cdi].fa; - } - curveDataElement.n = data[ci].stats[cdi].n; - } - if (data[ci].stats[cdi].obs_y) { - curveDataElement.oy = data[ci].stats[cdi].obs_y; - curveDataElement.on = data[ci].stats[cdi].obs_n; - } else { - curveDataElement.hitcount = data[ci].stats[cdi].hit_count; - curveDataElement.fcstcount = data[ci].stats[cdi].fcst_count; - } - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.gridscaleProb: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].y.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} probability bin`] = - data[ci].stats[cdi].bin_value; - curveDataElement["number of grid points"] = data[ci].stats[cdi].n_grid; - curveDataElement.n = data[ci].stats[cdi].n; - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.simpleScatter: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].y.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} bin value`] = - data[ci].stats[cdi].bin_value; - curveDataElement["x-stat"] = data[ci].stats[cdi].xstat; - curveDataElement["y-stat"] = data[ci].stats[cdi].ystat; - curveDataElement.n = data[ci].stats[cdi].n; - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.map: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - - var stats = {}; - stats.label = data[0].label; - stats["total number of obs"] = data[0].stats.reduce(function (prev, curr) { - return prev + curr.nTimes; - }, 0); - stats["mean difference"] = matsDataUtils.average(data[0].queryVal); - stats["standard deviation"] = matsDataUtils.stdev(data[0].queryVal); - stats["minimum time"] = data[0].stats.reduce(function (prev, curr) { - return prev < curr.min_time ? prev : curr.min_time; - }); - stats["minimum time"] = moment - .utc(stats["minimum time"] * 1000) - .format("YYYY-MM-DD HH:mm"); - stats["maximum time"] = data[0].stats.reduce(function (prev, curr) { - return prev > curr.max_time ? prev : curr.max_time; - }); - stats["maximum time"] = moment - .utc(stats["maximum time"] * 1000) - .format("YYYY-MM-DD HH:mm"); +// PUBLIC METHODS +// administration tools +const addSentAddress = new ValidatedMethod({ + name: "matsMethods.addSentAddress", + validate: new SimpleSchema({ + toAddress: { + type: String, + }, + }).validator(), + run(toAddress) { + if (!Meteor.userId()) { + throw new Meteor.Error(401, "not-logged-in"); + } + matsCollections.SentAddresses.upsert( + { + address: toAddress, + }, + { + address: toAddress, + userId: Meteor.userId(), + } + ); + return false; + }, +}); - returnData.stats[data[0].label] = stats; +// administation tool +const applyAuthorization = new ValidatedMethod({ + name: "matsMethods.applyAuthorization", + validate: new SimpleSchema({ + settings: { + type: Object, + blackbox: true, + }, + }).validator(), + run(settings) { + if (Meteor.isServer) { + let roles; + let roleName; + let authorization; - var curveData = []; // map of maps - isCTC = - data[0] !== undefined && - data[0].stats !== undefined && - data[0].stats[0] !== undefined && - data[0].stats[0].hit !== undefined; - for (var si = 0; si < data[0].siteName.length; si++) { - var curveDataElement = {}; - curveDataElement["site name"] = data[0].siteName[si]; - curveDataElement["number of times"] = data[0].stats[si].nTimes; - if (isCTC) { - curveDataElement.stat = data[0].queryVal[si]; - curveDataElement.hit = data[0].stats[si].hit; - curveDataElement.fa = data[0].stats[si].fa; - curveDataElement.miss = data[0].stats[si].miss; - curveDataElement.cn = data[0].stats[si].cn; - } else { - curveDataElement["start date"] = moment - .utc(data[0].stats[si].min_time * 1000) - .format("YYYY-MM-DD HH:mm"); - curveDataElement["end date"] = moment - .utc(data[0].stats[si].max_time * 1000) - .format("YYYY-MM-DD HH:mm"); - curveDataElement.stat = data[0].queryVal[si]; + const { userRoleName } = settings; + const { userRoleDescription } = settings; + const { authorizationRole } = settings; + const { newUserEmail } = settings; + const { existingUserEmail } = settings; + + if (authorizationRole) { + // existing role - the role roleName - no need to verify as the selection list came from the database + roleName = authorizationRole; + } else if (userRoleName && userRoleDescription) { + // possible new role - see if it happens to already exist + const role = matsCollections.Roles.findOne({ + name: userRoleName, + }); + if (role === undefined) { + // need to add new role using description + matsCollections.Roles.upsert( + { + name: userRoleName, + }, + { + $set: { + description: userRoleDescription, + }, } - curveData.push(curveDataElement); + ); + roleName = userRoleName; + } else { + // see if the description matches... + roleName = role.name; + const { description } = role; + if (description !== userRoleDescription) { + // have to update the description + matsCollections.Roles.upsert( + { + name: userRoleName, + }, + { + $set: { + description: userRoleDescription, + }, + } + ); } - returnData.data[data[0].label] = curveData; - break; - case matsTypes.PlotTypes.histogram: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.glob_mean; - stats["standard deviation"] = data[ci].glob_stats.glob_sd; - stats.n = data[ci].glob_stats.glob_n; - stats.minimum = data[ci].glob_stats.glob_min; - stats.maximum = data[ci].glob_stats.glob_max; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].x.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} bin range`] = - data[ci].bin_stats[cdi].binLabel; - curveDataElement.n = data[ci].bin_stats[cdi].bin_n; - curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; - curveDataElement["bin lower bound"] = data[ci].bin_stats[cdi].binLowBound; - curveDataElement["bin upper bound"] = data[ci].bin_stats[cdi].binUpBound; - curveDataElement["bin mean"] = data[ci].bin_stats[cdi].bin_mean; - curveDataElement["bin std dev"] = data[ci].bin_stats[cdi].bin_sd; - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; + } + } + // now we have a role roleName - now we need an email + if (existingUserEmail) { + // existing user - no need to verify as the selection list came from the database + // see if it already has the role + authorization = matsCollections.Authorization.findOne({ + email: existingUserEmail, + }); + roles = authorization.roles; + if (roles.indexOf(roleName) === -1) { + // have to add the role + if (roleName) { + roles.push(roleName); } - break; - case matsTypes.PlotTypes.ensembleHistogram: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves + matsCollections.Authorization.upsert( + { + email: existingUserEmail, + }, + { + $set: { + roles, + }, } - var stats = {}; - stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.dMean; - stats["standard deviation"] = data[ci].glob_stats.sd; - stats.n = data[ci].glob_stats.nGood; - stats.minimum = data[ci].glob_stats.minVal; - stats.maximum = data[ci].glob_stats.maxVal; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].x.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} bin`] = data[ci].x[cdi]; - curveDataElement.n = data[ci].bin_stats[cdi].bin_n; - curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; - curveData.push(curveDataElement); + ); + } + } else if (newUserEmail) { + // possible new authorization - see if it happens to exist + authorization = matsCollections.Authorization.findOne({ + email: newUserEmail, + }); + if (authorization !== undefined) { + // authorization exists - add role to roles if necessary + roles = authorization.roles; + if (roles.indexOf(roleName) === -1) { + // have to add the role + if (roleName) { + roles.push(roleName); } - returnData.data[data[ci].label] = curveData; + matsCollections.Authorization.upsert( + { + email: existingUserEmail, + }, + { + $set: { + roles, + }, + } + ); } - break; - case matsTypes.PlotTypes.contour: - case matsTypes.PlotTypes.contourDiff: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - var stats = {}; - stats.label = data[0].label; - stats["total number of points"] = data[0].glob_stats.n; - stats["mean stat"] = data[0].glob_stats.mean; - stats["minimum time"] = moment - .utc(data[0].glob_stats.minDate * 1000) - .format("YYYY-MM-DD HH:mm"); - stats["maximum time"] = moment - .utc(data[0].glob_stats.maxDate * 1000) - .format("YYYY-MM-DD HH:mm"); - - returnData.stats[data[0].label] = stats; - - var curveData = []; // array of maps - isCTC = - data[0] !== undefined && - data[0].hitTextOutput !== undefined && - data[0].hitTextOutput.length > 0; - for (var si = 0; si < data[0].xTextOutput.length; si++) { - var curveDataElement = {}; - curveDataElement.xVal = data[0].xTextOutput[si]; - curveDataElement.yVal = data[0].yTextOutput[si]; - curveDataElement.stat = data[0].zTextOutput[si]; - curveDataElement.N = data[0].nTextOutput[si]; - if (isCTC) { - curveDataElement.hit = data[0].hitTextOutput[si]; - curveDataElement.fa = data[0].faTextOutput[si]; - curveDataElement.miss = data[0].missTextOutput[si]; - curveDataElement.cn = data[0].cnTextOutput[si]; - } else { - curveDataElement["Start Date"] = moment - .utc(data[0].minDateTextOutput[si] * 1000) - .format("YYYY-MM-DD HH:mm"); - curveDataElement["End Date"] = moment - .utc(data[0].maxDateTextOutput[si] * 1000) - .format("YYYY-MM-DD HH:mm"); - } - curveData.push(curveDataElement); + } else { + // need a new authorization + roles = []; + if (roleName) { + roles.push(roleName); } - returnData.data[data[0].label] = curveData; - break; - case matsTypes.PlotTypes.scatter2d: - var returnData = {}; // returns a map of arrays of maps - var firstBestFitIndex = -1; - var bestFitIndexes = {}; - for (var ci = 0; ci < data.length; ci++) { - if (ci === firstBestFitIndex) { - break; // best fit curves are at the end so do not do further processing - } - var curveData = data[ci]; - // look for a best fit curve - only have to look at curves with higher index than this one - const bestFitIndex = -1; - for (let cbi = ci + 1; cbi < data.length; cbi++) { - if ( - data[cbi].label.indexOf(curveData.label) !== -1 && - data[cbi].label.indexOf("-best fit") !== -1 - ) { - bestFitIndexes[ci] = cbi; - if (firstBestFitIndex === -1) { - firstBestFitIndex = cbi; - } - break; - } - } - const curveTextData = []; - for (var cdi = 0; cdi < curveData.data.length; cdi++) { - const element = {}; - element.xAxis = curveData.data[cdi][0]; - element.yAxis = curveData.data[cdi][1]; - if (bestFitIndexes[ci] === undefined) { - element["best fit"] = "none;"; - } else { - element["best fit"] = data[bestFitIndexes[ci]].data[cdi][1]; + if (newUserEmail) { + matsCollections.Authorization.upsert( + { + email: newUserEmail, + }, + { + $set: { + roles, + }, } - curveTextData.push(element); - } - returnData[curveData.label] = curveTextData; + ); + } + } + } + } + return false; + }, +}); + +// database controls +const applyDatabaseSettings = new ValidatedMethod({ + name: "matsMethods.applyDatabaseSettings", + validate: new SimpleSchema({ + settings: { + type: Object, + blackbox: true, + }, + }).validator(), + + run(settings) { + if (Meteor.isServer) { + if (settings.name) { + matsCollections.Databases.upsert( + { + name: settings.name, + }, + { + $set: { + name: settings.name, + role: settings.role, + status: settings.status, + host: settings.host, + database: settings.database, + user: settings.user, + password: settings.password, + }, } - break; - default: - return undefined; + ); } - returnData.dsiRealPageIndex = dsiRealPageIndex; - returnData.dsiTextDirection = dsiTextDirection; - return returnData; - } catch (error) { - throw new Meteor.Error( - `Error in _getFlattenedResultData function: ${error.message}` - ); } - } -}; - -// private method for getting pagenated data -// a newPageIndex of -1000 means get all the data (used for export) -// a newPageIndex of -2000 means get just the last page -const _getPagenatedData = function (rky, p, np) { - if (Meteor.isServer) { - const key = rky; - const myPageIndex = p; - const newPageIndex = np; - let ret; - let rawReturn; + return false; + }, +}); - try { - const result = matsCache.getResult(key); - rawReturn = result === undefined ? undefined : result.result; // getResult structure is {key:something, result:resultObject} - } catch (e) { - console.log("_getPagenatedData: Error - ", e); - return undefined; +// administration tools +const deleteSettings = new ValidatedMethod({ + name: "matsMethods.deleteSettings", + validate: new SimpleSchema({ + name: { + type: String, + }, + }).validator(), + run(params) { + if (!Meteor.userId()) { + throw new Meteor.Error("not-logged-in"); } - ret = rawReturn === undefined ? undefined : JSON.parse(JSON.stringify(rawReturn)); - let start; - let end; - let direction = 1; - if (newPageIndex === -1000) { - // all the data - start = 0; - end = Number.MAX_VALUE; - } else if (newPageIndex === -2000) { - // just the last page - start = -2000; - direction = -1; - } else if (myPageIndex <= newPageIndex) { - // proceed forward - start = (newPageIndex - 1) * 100; - end = newPageIndex * 100; - } else { - // move back - direction = -1; - start = newPageIndex * 100; - end = (newPageIndex + 1) * 100; + if (Meteor.isServer) { + matsCollections.CurveSettings.remove({ + name: params.name, + }); } + }, +}); - let dsiStart; - let dsiEnd; - for (let csi = 0; csi < ret.data.length; csi++) { - if (!ret.data[csi].x || ret.data[csi].x.length <= 100) { - continue; // don't bother pagenating datasets less than or equal to a page - ret is rawReturn - } - dsiStart = start; - dsiEnd = end; - if (dsiStart > ret.data[csi].x.length || dsiStart === -2000) { - // show the last page if we either requested it specifically or are trying to navigate past it - dsiStart = Math.floor(rawReturn.data[csi].x.length / 100) * 100; - dsiEnd = rawReturn.data[csi].x.length; - if (dsiEnd === dsiStart) { - // make sure the last page isn't empty--if rawReturn.data[csi].data.length/100 produces a whole number, - // dsiStart and dsiEnd would be the same. This makes sure that the last full page is indeed the last page, without a phantom empty page afterwards - dsiStart = dsiEnd - 100; - } - } - if (dsiStart < 0) { - // show the first page if we are trying to navigate before it - dsiStart = 0; - dsiEnd = 100; - } - if (dsiEnd < dsiStart) { - // make sure that the end is after the start - dsiEnd = dsiStart + 100; - } - if (dsiEnd > ret.data[csi].x.length) { - // make sure we don't request past the end -- if results are one page, this should convert the - // start and end from 0 and 100 to 0 and whatever the end is. - dsiEnd = ret.data[csi].x.length; - } - ret.data[csi].x = rawReturn.data[csi].x.slice(dsiStart, dsiEnd); - ret.data[csi].y = rawReturn.data[csi].y.slice(dsiStart, dsiEnd); - ret.data[csi].stats = rawReturn.data[csi].stats.slice(dsiStart, dsiEnd); - ret.data[csi].glob_stats = rawReturn.data[csi].glob_stats; +// drop a single instance of a scorecard +const dropScorecardInstance = new ValidatedMethod({ + name: "matsMethods.dropScorecardInstance", + validate: new SimpleSchema({ + userName: { + type: String, + }, + name: { + type: String, + }, + submittedTime: { + type: String, + }, + processedAt: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + return dropThisScorecardInstance( + params.userName, + params.name, + params.submittedTime, + params.processedAt + ); } + return null; + }, +}); - if (direction === 1) { - ret.dsiRealPageIndex = Math.floor(dsiEnd / 100); - } else { - ret.dsiRealPageIndex = Math.floor(dsiStart / 100); +// administration tools +const emailImage = new ValidatedMethod({ + name: "matsMethods.emailImage", + validate: new SimpleSchema({ + imageStr: { + type: String, + }, + toAddress: { + type: String, + }, + subject: { + type: String, + }, + }).validator(), + run(params) { + const { imageStr } = params; + const { toAddress } = params; + const { subject } = params; + if (!Meteor.userId()) { + throw new Meteor.Error(401, "not-logged-in"); } - ret.dsiTextDirection = direction; - return ret; - } -}; + const fromAddress = Meteor.user().services.google.email; + // these come from google - see + // http://masashi-k.blogspot.fr/2013/06/sending-mail-with-gmail-using-xoauth2.html + // http://stackoverflow.com/questions/24098461/nodemailer-gmail-what-exactly-is-a-refresh-token-and-how-do-i-get-one/24123550 -// private define a middleware for refreshing the metadata -const _refreshMetadataMWltData = function (params, req, res, next) { - if (Meteor.isServer) { - console.log("Server route asked to refresh metadata"); + // the gmail account for the credentials is mats.mail.daemon@gmail.com - pwd mats2015! + // var clientId = "339389735380-382sf11aicmgdgn7e72p4end5gnm9sad.apps.googleusercontent.com"; + // var clientSecret = "7CfNN-tRl5QAL595JTW2TkRl"; + // var refresh_token = "1/PDql7FR01N2gmq5NiTfnrT-OlCYC3U67KJYYDNPeGnA"; + const credentials = matsCollections.Credentials.findOne( + { + name: "oauth_google", + }, + { + clientId: 1, + clientSecret: 1, + refresh_token: 1, + } + ); + const { clientId } = credentials; + const { clientSecret } = credentials; + const refreshToken = credentials.refresh_token; + let smtpTransporter; try { - console.log("GUI asked to refresh metadata"); - _checkMetaDataRefresh(); + const Nodemailer = require("nodemailer"); + smtpTransporter = Nodemailer.createTransport("SMTP", { + service: "Gmail", + auth: { + XOAuth2: { + user: "mats.gsl@noaa.gov", + clientId, + clientSecret, + refreshToken, + }, + }, + }); } catch (e) { - console.log(e); - res.end( - `` + - `

refreshMetadata Failed!

` + - `

${e.message}

` + - `` - ); + throw new Meteor.Error(401, `Transport error ${e.message()}`); } - res.end("

refreshMetadata Done!

"); - } -}; + try { + const mailOptions = { + sender: fromAddress, + replyTo: fromAddress, + from: fromAddress, + to: toAddress, + subject, + attachments: [ + { + filename: "graph.png", + contents: Buffer.from(imageStr.split("base64,")[1], "base64"), + }, + ], + }; -// private middleware for causing the scorecard to refresh its mongo collection for a given document -const _refreshScorecard = function (params, req, res, next) { - if (Meteor.isServer) { - const docId = decodeURIComponent(params.docId); - // get userName, name, submitted, processedAt from id - // SC:anonymous--submitted:20230322230435--1block:0:02/19/2023_20_00_-_03/21/2023_20_00 - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardData: No cbScorecardPool defined"); - } - const statement = `SELECT sc.* - From - vxdata._default.SCORECARD sc - WHERE - sc.id='${docId}';`; - cbScorecardPool - .queryCB(statement) - .then((result) => { - // insert this result into the mongo Scorecard collection - createdAt is used for TTL - // created at gets updated each display even if it already existed. - // TTL is 24 hours - if (typeof result === "string") { - throw new Error(`Error from couchbase query - ${result}`); - } else if (result[0] === undefined) { - throw new Error("Error from couchbase query - document not found"); - } else { - matsCollections.Scorecard.upsert( - { - "scorecard.userName": result[0].userName, - "scorecard.name": result[0].name, - "scorecard.submitted": result[0].submitted, - "scorecard.processedAt": result[0].processedAt, - }, - { - $set: { - createdAt: new Date(), - scorecard: result[0], - }, - } + smtpTransporter.sendMail(mailOptions, function (error, response) { + if (error) { + console.log( + `smtpTransporter error ${error} from:${fromAddress} to:${toAddress}` ); + } else { + console.log(`${response} from:${fromAddress} to:${toAddress}`); } - res.end("

refreshScorecard Done!

"); - }) - .catch((err) => { - res.end( - `` + - `

refreshScorecard Failed!

` + - `

${err.message}

` + - `` - ); + smtpTransporter.close(); }); - } -}; + } catch (e) { + throw new Meteor.Error(401, `Send error ${e.message()}`); + } + return false; + }, +}); + +// administation tool +const getAuthorizations = new ValidatedMethod({ + name: "matsMethods.getAuthorizations", + validate: new SimpleSchema({}).validator(), + run() { + let roles = []; + if (Meteor.isServer) { + const userEmail = Meteor.user().services.google.email.toLowerCase(); + roles = matsCollections.Authorization.findOne({ + email: userEmail, + }).roles; + } + return roles; + }, +}); -const _setStatusScorecard = function (params, req, res, next) { - if (Meteor.isServer) { - const docId = decodeURIComponent(params.docId); - let body = ""; - req.on( - "data", - Meteor.bindEnvironment(function (data) { - body += data; - }) - ); +// administration tool - req.on( - "end", - Meteor.bindEnvironment(function () { - // console.log(body); - try { - const doc = JSON.parse(body); - const { status } = doc; - const { error } = doc; - const found = matsCollections.Scorecard.find({ id: docId }).fetch(); - if (found.length === 0) { - throw new Error("Error from scorecard lookup - document not found"); +const getRunEnvironment = new ValidatedMethod({ + name: "matsMethods.getRunEnvironment", + validate: new SimpleSchema({}).validator(), + run() { + return Meteor.settings.public.run_environment; + }, +}); + +const getDefaultGroupList = new ValidatedMethod({ + name: "matsMethods.getDefaultGroupList", + validate: new SimpleSchema({}).validator(), + run() { + return matsTypes.DEFAULT_GROUP_LIST; + }, +}); + +// retrieves the saved query results (or downsampled results) +const getGraphData = new ValidatedMethod({ + name: "matsMethods.getGraphData", + validate: new SimpleSchema({ + plotParams: { + type: Object, + blackbox: true, + }, + plotType: { + type: String, + }, + expireKey: { + type: Boolean, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + const plotGraphFunction = matsCollections.PlotGraphFunctions.findOne({ + plotType: params.plotType, + }); + const { dataFunction } = plotGraphFunction; + let ret; + try { + const hash = require("object-hash"); + const key = hash(params.plotParams); + if (process.env.NODE_ENV === "development" || params.expireKey) { + matsCache.expireKey(key); + } + const results = matsCache.getResult(key); + if (results === undefined) { + // results aren't in the cache - need to process data routine + const Future = require("fibers/future"); + const future = new Future(); + global[dataFunction](params.plotParams, function (result) { + ret = saveResultData(result); + future.return(ret); + }); + return future.wait(); + } + // results were already in the matsCache (same params and not yet expired) + // are results in the downsampled collection? + const dsResults = DownSampleResults.findOne( + { + key, + }, + {}, + { + disableOplog: true, } - matsCollections.Scorecard.upsert( + ); + if (dsResults !== undefined) { + // results are in the mongo cache downsampled collection - returned the downsampled graph data + ret = dsResults; + // update the expire time in the downsampled collection - this requires a new Date + DownSampleResults.rawCollection().update( { - id: docId, + key, }, { $set: { - status, + createdAt: new Date(), }, } ); - // set error if there is one somehow. (use the session?) - res.end("

setScorecardStatus Done!

"); - } catch (err) { - res.statusCode = 400; - res.end( - `` + - `

setScorecardStatus Failed!

` + - `

${err.message}

` + - `` + } else { + ret = results; // {key:someKey, result:resultObject} + // refresh expire time. The only way to perform a refresh on matsCache is to re-save the result. + matsCache.storeResult(results.key, results); + } + const sizeof = require("object-sizeof"); + console.log("result.data size is ", sizeof(results)); + return ret; + } catch (dataFunctionError) { + if (dataFunctionError.toLocaleString().indexOf("INFO:") !== -1) { + throw new Meteor.Error(dataFunctionError.message); + } else { + throw new Meteor.Error( + `Error in getGraphData function:${dataFunction} : ${dataFunctionError.message}` ); } - }) - ); - } -}; + } + } + return null; + }, +}); -// private save the result from the query into mongo and downsample if that result's size is greater than 1.2Mb -const _saveResultData = function (result) { - if (Meteor.isServer) { - const sizeof = require("object-sizeof"); - const hash = require("object-hash"); - const key = hash(result.basis.plotParams); - const threshold = 1200000; - let ret = {}; - try { - const dSize = sizeof(result.data); - // console.log("result.basis.data size is ", dSize); - // TimeSeries and DailyModelCycle are the only plot types that require downSampling - if ( - dSize > threshold && - (result.basis.plotParams.plotTypes.TimeSeries || - result.basis.plotParams.plotTypes.DailyModelCycle) - ) { - // greater than threshold need to downsample - // downsample and save it in DownSampleResult - console.log("DownSampling"); - const downsampler = require("downsample-lttb"); - let totalPoints = 0; - for (let di = 0; di < result.data.length; di++) { - totalPoints += result.data[di].x_epoch.length; - } - const allowedNumberOfPoints = (threshold / dSize) * totalPoints; - const downSampleResult = - result === undefined ? undefined : JSON.parse(JSON.stringify(result)); - for (var ci = 0; ci < result.data.length; ci++) { - const dsData = {}; - const xyDataset = result.data[ci].x_epoch.map(function (d, index) { - return [result.data[ci].x_epoch[index], result.data[ci].y[index]]; - }); - const ratioTotalPoints = xyDataset.length / totalPoints; - const myAllowedPoints = Math.round(ratioTotalPoints * allowedNumberOfPoints); - // downsample the array - var downsampledSeries; - if (myAllowedPoints < xyDataset.length && xyDataset.length > 2) { - downsampledSeries = downsampler.processData(xyDataset, myAllowedPoints); - // replace the y attributes (tooltips etc.) with the y attributes from the nearest x - let originalIndex = 0; - // skip through the original dataset capturing each downSampled data point - const arrayKeys = []; - const nonArrayKeys = []; - const keys = Object.keys(result.data[ci]); - for (var ki = 0; ki < keys.length; ki++) { - if (keys[ki] !== "x_epoch") { - if (Array.isArray(result.data[ci][keys[ki]])) { - arrayKeys.push(keys[ki]); - dsData[keys[ki]] = []; - } else { - nonArrayKeys.push(keys[ki]); - } - } - } - // We only ever downsample series plots - never profiles and series plots only ever have error_y arrays. - // This is a little hacky but what is happening is we putting error_y.array on the arrayKeys list so that it gets its - // downsampled equivalent values. - for (ki = 0; ki < nonArrayKeys.length; ki++) { - dsData[nonArrayKeys[ki]] = JSON.parse( - JSON.stringify(result.data[ci][nonArrayKeys[ki]]) - ); - } - // remove the original error_y array data. - dsData.error_y.array = []; - for (let dsi = 0; dsi < downsampledSeries.length; dsi++) { - while ( - originalIndex < result.data[ci].x_epoch.length && - result.data[ci].x_epoch[originalIndex] < downsampledSeries[dsi][0] - ) { - originalIndex++; - } - // capture the stuff related to this downSampled data point (downSampled data points are always a subset of original data points) - for (ki = 0; ki < arrayKeys.length; ki++) { - dsData[arrayKeys[ki]][dsi] = - result.data[ci][arrayKeys[ki]][originalIndex]; - } - dsData.error_y.array[dsi] = result.data[ci].error_y.array[originalIndex]; - } - // add downsampled annotation to curve options - downSampleResult[ci] = dsData; - downSampleResult[ci].annotation += " **DOWNSAMPLED**"; - } else { - downSampleResult[ci] = result.data[ci]; +// retrieves the saved query results (or downsampled results) for a specific key +const getGraphDataByKey = new ValidatedMethod({ + name: "matsMethods.getGraphDataByKey", + validate: new SimpleSchema({ + resultKey: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + let ret; + const key = params.resultKey; + try { + const dsResults = DownSampleResults.findOne( + { + key, + }, + {}, + { + disableOplog: true, } - downSampleResult.data[ci] = downSampleResult[ci]; + ); + if (dsResults !== undefined) { + ret = dsResults; + } else { + ret = matsCache.getResult(key); // {key:someKey, result:resultObject} } - DownSampleResults.rawCollection().insert({ - createdAt: new Date(), - key, - result: downSampleResult, - }); // createdAt ensures expiration set in mats-collections - ret = { - key, - result: downSampleResult, - }; - } else { - ret = { + const sizeof = require("object-sizeof"); + console.log("getGraphDataByKey results size is ", sizeof(dsResults)); + return ret; + } catch (error) { + throw new Meteor.Error( + `Error in getGraphDataByKey function:${key} : ${error.message}` + ); + } + } + return null; + }, +}); + +const getLayout = new ValidatedMethod({ + name: "matsMethods.getLayout", + validate: new SimpleSchema({ + resultKey: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + let ret; + const key = params.resultKey; + try { + ret = LayoutStoreCollection.rawCollection().findOne({ key, - result, - }; + }); + return ret; + } catch (error) { + throw new Meteor.Error(`Error in getLayout function:${key} : ${error.message}`); } - // save original dataset in the matsCache - if ( - result.basis.plotParams.plotTypes.TimeSeries || - result.basis.plotParams.plotTypes.DailyModelCycle - ) { - for (var ci = 0; ci < result.data.length; ci++) { - delete result.data[ci].x_epoch; // we only needed this as an index for downsampling - } + } + return null; + }, +}); + +const getScorecardSettings = new ValidatedMethod({ + name: "matsMethods.getScorecardSettings", + validate: new SimpleSchema({ + settingsKey: { + type: String, + }, + }).validator(), + async run(params) { + if (Meteor.isServer) { + const key = params.settingsKey; + try { + // global cbScorecardSettingsPool + const rv = await cbScorecardSettingsPool.getCB(key); + return { scorecardSettings: rv.content }; + } catch (error) { + throw new Meteor.Error( + `Error in getScorecardSettings function:${key} : ${error.message}` + ); + } + } + return null; + }, +}); + +const getPlotParamsFromScorecardInstance = new ValidatedMethod({ + name: "matsMethods.getPlotParamsFromScorecardInstance", + validate: new SimpleSchema({ + userName: { + type: String, + }, + name: { + type: String, + }, + submitted: { + type: String, + }, + processedAt: { + type: String, + }, + }).validator(), + run(params) { + try { + if (Meteor.isServer) { + return getThesePlotParamsFromScorecardInstance( + params.userName, + params.name, + params.submitted, + params.processedAt + ); } - matsCache.storeResult(key, { - key, - result, - }); // lifespan is handled by lowDb (internally) in matscache } catch (error) { - if (error.toLocaleString().indexOf("larger than the maximum size") !== -1) { - throw new Meteor.Error(": Requesting too much data... try averaging"); + throw new Meteor.Error( + `Error in getPlotParamsFromScorecardInstance function:${error.message}` + ); + } + return null; + }, +}); + +/* +getPlotResult is used by the graph/text_*_output templates which are used to display textual results. +Because the data isn't being rendered graphically this data is always full size, i.e. NOT downsampled. +That is why it only finds it in the Result file cache, never the DownSampleResult collection. + +Because the dataset can be so large ... e.g. megabytes the data retrieval is pagenated. The index is +applied to the underlying datasets.The data gets stripped down and flattened to only contain the data neccesary for text presentation. +A new page index of -1000 means get all the data i.e. no pagenation. + */ +const getPlotResult = new ValidatedMethod({ + name: "matsMethods.getPlotResult", + validate: new SimpleSchema({ + resultKey: { + type: String, + }, + pageIndex: { + type: Number, + }, + newPageIndex: { + type: Number, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + const rKey = params.resultKey; + const pi = params.pageIndex; + const npi = params.newPageIndex; + let ret = {}; + try { + ret = getFlattenedResultData(rKey, pi, npi); + } catch (e) { + console.log(e); } + return ret; } - return ret; - } -}; + return null; + }, +}); -// Utility method for writing out the meteor.settings file -const _write_settings = function (settings, appName) { - const fs = require("fs"); - let settingsPath = process.env.METEOR_SETTINGS_DIR; - if (!settingsPath) { - console.log( - "environment var METEOR_SETTINGS_DIR is undefined: setting it to /usr/app/settings" - ); - settingsPath = "/usr/app/settings"; - } - if (!fs.existsSync(settingsPath)) { - fs.mkdirSync(settingsPath, { - recursive: true, - }); - } - let appSettings = {}; - let newSettings = {}; - try { - const appSettingsData = fs.readFileSync(`${settingsPath}/${appName}/settings.json`); - appSettings = JSON.parse(appSettingsData); - } catch (e) { - appSettings = { - private: {}, - public: {}, - }; - } - newSettings = settings; - // Merge settings into appSettings - newSettings.private = { - ...appSettings.private, - ...settings.private, - }; - newSettings.public = { - ...appSettings.public, - ...settings.public, - }; - // write the settings file - const jsonSettings = JSON.stringify(newSettings, null, 2); - // console.log (jsonSettings); - fs.writeFileSync(`${settingsPath}/${appName}/settings.json`, jsonSettings, { - encoding: "utf8", - flag: "w", - }); -}; -// return the scorecard for the provided selectors -const _getScorecardData = async function (userName, name, submitted, processedAt) { - try { - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardData: No cbScorecardPool defined"); +const getReleaseNotes = new ValidatedMethod({ + name: "matsMethods.getReleaseNotes", + validate: new SimpleSchema({}).validator(), + run() { + // return Assets.getText('public/MATSReleaseNotes.html'); + // } + if (Meteor.isServer) { + const Future = require("fibers/future"); + const fse = require("fs-extra"); + const dFuture = new Future(); + let fData; + let file; + if (process.env.NODE_ENV === "development") { + file = `${process.env.PWD}/.meteor/local/build/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; + } else { + file = `${process.env.PWD}/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; + } + try { + fse.readFile(file, "utf8", function (err, data) { + if (err) { + fData = err.message; + dFuture.return(); + } else { + fData = data; + dFuture.return(); + } + }); + } catch (e) { + fData = e.message; + dFuture.return(); + } + dFuture.wait(); + return fData; } - const statement = `SELECT sc.* - From - vxdata._default.SCORECARD sc - WHERE - sc.type='SC' - AND sc.userName='${userName}' - AND sc.name='${name}' - AND sc.processedAt=${processedAt} - AND sc.submitted=${submitted};`; - const result = await cbScorecardPool.queryCBWithConsistency(statement); - if (typeof result === "string" && result.indexOf("ERROR")) { - throw new Meteor.Error(result); + return null; + }, +}); + +const setCurveParamDisplayText = new ValidatedMethod({ + name: "matsMethods.setCurveParamDisplayText", + validate: new SimpleSchema({ + paramName: { + type: String, + }, + newText: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + return matsCollections[params.paramName].update( + { name: params.paramName }, + { $set: { controlButtonText: params.newText } } + ); + } + return null; + }, +}); + +const getScorecardData = new ValidatedMethod({ + name: "matsMethods.getScorecardData", + validate: new SimpleSchema({ + userName: { + type: String, + }, + name: { + type: String, + }, + submitted: { + type: String, + }, + processedAt: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + return getThisScorecardData( + params.userName, + params.name, + params.submitted, + params.processedAt + ); + } + return null; + }, +}); + +const getScorecardInfo = new ValidatedMethod({ + name: "matsMethods.getScorecardInfo", + validate: new SimpleSchema({}).validator(), + run() { + if (Meteor.isServer) { + return getThisScorecardInfo(); + } + return null; + }, +}); + +// administration tool +const getUserAddress = new ValidatedMethod({ + name: "matsMethods.getUserAddress", + validate: new SimpleSchema({}).validator(), + run() { + if (Meteor.isServer) { + return Meteor.user().services.google.email.toLowerCase(); } - // insert this result into the mongo Scorecard collection - createdAt is used for TTL - // created at gets updated each display even if it already existed. - // TTL is 24 hours - matsCollections.Scorecard.upsert( - { - "scorecard.userName": result[0].userName, - "scorecard.name": result[0].name, - "scorecard.submitted": result[0].submitted, - "scorecard.processedAt": result[0].processedAt, - }, - { - $set: { - createdAt: new Date(), - scorecard: result[0], - }, - } - ); - const docID = matsCollections.Scorecard.findOne( - { - "scorecard.userName": result[0].userName, - "scorecard.name": result[0].name, - "scorecard.submitted": result[0].submitted, - "scorecard.processedAt": result[0].processedAt, - }, - { _id: 1 } - )._id; - // no need to return the whole thing, just the identifying fields - // and the ID. The app will find the whole thing in the mongo collection. - return { scorecard: result[0], docID }; - } catch (err) { - console.log(`_getScorecardData error : ${err.message}`); - return { - error: err.message, - }; - } -}; + return null; + }, +}); -// return the scorecard status information from the couchbase database -const _getScorecardInfo = async function () { - try { - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardInfo: No cbScorecardPool defined"); +// app utility +const insertColor = new ValidatedMethod({ + name: "matsMethods.insertColor", + validate: new SimpleSchema({ + newColor: { + type: String, + }, + insertAfterIndex: { + type: Number, + }, + }).validator(), + run(params) { + if (params.newColor === "rgb(255,255,255)") { + return false; } + const colorScheme = matsCollections.ColorScheme.findOne({}); + colorScheme.colors.splice(params.insertAfterIndex, 0, params.newColor); + matsCollections.update({}, colorScheme); + return false; + }, +}); - const statement = `SELECT - sc.id, - sc.userName, - sc.name, - sc.status, - sc.processedAt as processedAt, - sc.submitted, - sc.dateRange - From - vxdata._default.SCORECARD sc - WHERE - sc.type='SC';`; - const result = await cbScorecardPool.queryCBWithConsistency(statement); - scMap = {}; - result.forEach(function (elem) { - if (!Object.keys(scMap).includes(elem.userName)) { - scMap[elem.userName] = {}; +// administration tool +const readFunctionFile = new ValidatedMethod({ + name: "matsMethods.readFunctionFile", + validate: new SimpleSchema({ + file: { + type: String, + }, + type: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + const future = require("fibers/future"); + const fse = require("fs-extra"); + let path = ""; + let fData; + if (params.type === "data") { + path = `/web/static/dataFunctions/${params.file}`; + console.log(`exporting data file: ${path}`); + } else if (params.type === "graph") { + path = `/web/static/displayFunctions/${params.file}`; + console.log(`exporting graph file: ${path}`); + } else { + return "error - wrong type"; } - userElem = scMap[elem.userName]; - if (!Object.keys(userElem).includes(elem.name)) { - userElem[elem.name] = {}; + fse.readFile(path, function (err, data) { + if (err) throw err; + fData = data.toString(); + future.return(fData); + }); + return future.wait(); + } + return null; + }, +}); + +// refreshes the metadata for the app that's running +const refreshMetaData = new ValidatedMethod({ + name: "matsMethods.refreshMetaData", + validate: new SimpleSchema({}).validator(), + run() { + if (Meteor.isServer) { + try { + // console.log("GUI asked to refresh metadata"); + checkMetaDataRefresh(); + } catch (e) { + console.log(e); + throw new Meteor.Error("Server error: ", e.message); } - nameElem = userElem[elem.name]; - if (!Object.keys(nameElem).includes(elem.submited)) { - nameElem[elem.submitted] = {}; + } + return metaDataTableUpdates.find({}).fetch(); + }, +}); + +// administation tool +const removeAuthorization = new ValidatedMethod({ + name: "matsMethods.removeAuthorization", + validate: new SimpleSchema({ + settings: { + type: Object, + blackbox: true, + }, + }).validator(), + run(settings) { + if (Meteor.isServer) { + let email; + let roleName; + const { userRoleName } = settings; + const { authorizationRole } = settings; + const { newUserEmail } = settings; + const { existingUserEmail } = settings; + if (authorizationRole) { + // existing role - the role roleName - no need to verify as the selection list came from the database + roleName = authorizationRole; + } else if (userRoleName) { + roleName = userRoleName; + } + if (existingUserEmail) { + email = existingUserEmail; + } else { + email = newUserEmail; } - submittedElem = nameElem[elem.submitted]; - submittedElem[elem.processedAt] = { - id: elem.id, - status: elem.status, - submitted: elem.submitted, - }; - }); - return scMap; - } catch (err) { - console.log(`_getScorecardInfo error : ${err.message}`); - return { - error: err.message, - }; - } -}; -const _getPlotParamsFromScorecardInstance = async function ( - userName, - name, - submitted, - processedAt -) { - try { - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardInfo: No cbScorecardPool defined"); - } - const statement = `SELECT sc.plotParams - From - vxdata._default.SCORECARD sc - WHERE - sc.type='SC' - AND sc.userName='${userName}' - AND sc.name='${name}' - AND sc.processedAt=${processedAt} - AND sc.submitted=${submitted};`; - const result = await cbScorecardPool.queryCBWithConsistency(statement); - if (typeof result === "string" && result.indexOf("ERROR")) { - throw new Meteor.Error(result); + // if user and role remove the role from the user + if (email && roleName) { + matsCollections.Authorization.update( + { + email, + }, + { + $pull: { + roles: roleName, + }, + } + ); + } + // if user and no role remove the user + if (email && !roleName) { + matsCollections.Authorization.remove({ + email, + }); + } + // if role and no user remove role and remove role from all users + if (roleName && !email) { + // remove the role + matsCollections.Roles.remove({ + name: roleName, + }); + // remove the roleName role from all the authorizations + matsCollections.Authorization.update( + { + roles: roleName, + }, + { + $pull: { + roles: roleName, + }, + }, + { + multi: true, + } + ); + } } - return result[0]; - } catch (err) { - console.log(`_getPlotParamsFromScorecardInstance error : ${err.message}`); - return { - error: err.message, - }; - } -}; + return false; + }, +}); + +// app utility +const removeColor = new ValidatedMethod({ + name: "matsMethods.removeColor", + validate: new SimpleSchema({ + removeColor: { + type: String, + }, + }).validator(), + run(params) { + const colorScheme = matsCollections.ColorScheme.findOne({}); + const removeIndex = colorScheme.colors.indexOf(params.removeColor); + colorScheme.colors.splice(removeIndex, 1); + matsCollections.ColorScheme.update({}, colorScheme); + return false; + }, +}); -// PUBLIC METHODS -// administration tools -const addSentAddress = new ValidatedMethod({ - name: "matsMethods.addSentAddress", +// database controls +const removeDatabase = new ValidatedMethod({ + name: "matsMethods.removeDatabase", validate: new SimpleSchema({ - toAddress: { + dbName: { type: String, }, }).validator(), - run(toAddress) { - if (!Meteor.userId()) { - throw new Meteor.Error(401, "not-logged-in"); + run(dbName) { + if (Meteor.isServer) { + matsCollections.Databases.remove({ + name: dbName, + }); } - matsCollections.SentAddresses.upsert( - { - address: toAddress, - }, - { - address: toAddress, - userId: Meteor.userId(), - } - ); - return false; }, }); -// administation tool -const applyAuthorization = new ValidatedMethod({ - name: "matsMethods.applyAuthorization", +const applySettingsData = new ValidatedMethod({ + name: "matsMethods.applySettingsData", validate: new SimpleSchema({ settings: { type: Object, blackbox: true, }, }).validator(), - run(settings) { + // this method forces a restart on purpose. We do not want retries + applyOptions: { + noRetry: true, + }, + run(settingsParam) { if (Meteor.isServer) { - let roles; - let roleName; - let authorization; - - const { userRoleName } = settings; - const { userRoleDescription } = settings; - const { authorizationRole } = settings; - const { newUserEmail } = settings; - const { existingUserEmail } = settings; + // Read the existing settings file + const { settings } = settingsParam; + console.log( + "applySettingsData - matsCollections.appName.findOne({}) is ", + matsCollections.appName.findOne({}) + ); + const { appName } = matsCollections.Settings.findOne({}); + writeSettings(settings, appName); + // in development - when being run by meteor, this should force a restart of the app. + // in case I am in a container - exit and force a reload + console.log( + `applySettingsData - process.env.NODE_ENV is: ${process.env.NODE_ENV}` + ); + if (process.env.NODE_ENV !== "development") { + console.log("applySettingsData - exiting after writing new Settings"); + process.exit(0); + } + } + }, +}); - if (authorizationRole) { - // existing role - the role roleName - no need to verify as the selection list came from the database - roleName = authorizationRole; - } else if (userRoleName && userRoleDescription) { - // possible new role - see if it happens to already exist - const role = matsCollections.Roles.findOne({ - name: userRoleName, - }); - if (role === undefined) { - // need to add new role using description - matsCollections.Roles.upsert( - { - name: userRoleName, - }, - { - $set: { - description: userRoleDescription, - }, - } - ); - roleName = userRoleName; +// makes sure all of the parameters display appropriate selections in relation to one another +// for default settings ... +const resetApp = async function (appRef) { + if (Meteor.isServer) { + const metaDataTableRecords = appRef.appMdr; + const { appPools } = appRef; + const type = appRef.appType; + const scorecard = Meteor.settings.public.scorecard + ? Meteor.settings.public.scorecard + : false; + const dbType = appRef.dbType ? appRef.dbType : matsTypes.DbTypes.mysql; + const appName = Meteor.settings.public.app ? Meteor.settings.public.app : "unnamed"; + const appTitle = Meteor.settings.public.title + ? Meteor.settings.public.title + : "Unnamed App"; + const appGroup = Meteor.settings.public.group + ? Meteor.settings.public.group + : "Misc. Apps"; + const thresholdUnits = Meteor.settings.public.threshold_units + ? Meteor.settings.public.threshold_units + : {}; + let appDefaultGroup = ""; + let appDefaultDB = ""; + let appDefaultModel = ""; + let appColor; + switch (type) { + case matsTypes.AppTypes.metexpress: + appColor = Meteor.settings.public.color + ? Meteor.settings.public.color + : "darkorchid"; + appDefaultGroup = Meteor.settings.public.default_group + ? Meteor.settings.public.default_group + : "NO GROUP"; + appDefaultDB = Meteor.settings.public.default_db + ? Meteor.settings.public.default_db + : "mv_default"; + appDefaultModel = Meteor.settings.public.default_model + ? Meteor.settings.public.default_model + : "Default"; + break; + case matsTypes.AppTypes.mats: + default: + if (dbType === matsTypes.DbTypes.couchbase) { + appColor = "#33abbb"; } else { - // see if the description matches... - roleName = role.name; - const { description } = role; - if (description !== userRoleDescription) { - // have to update the description - matsCollections.Roles.upsert( - { - name: userRoleName, - }, - { - $set: { - description: userRoleDescription, - }, - } - ); - } + appColor = Meteor.settings.public.color + ? Meteor.settings.public.color + : "#3366bb"; } + break; + } + const appTimeOut = Meteor.settings.public.mysql_wait_timeout + ? Meteor.settings.public.mysql_wait_timeout + : 300; + let depEnv = process.env.NODE_ENV; + const curveParams = curveParamsByApp[Meteor.settings.public.app]; + let appsToScore; + if (Meteor.settings.public.scorecard) { + appsToScore = Meteor.settings.public.apps_to_score + ? Meteor.settings.public.apps_to_score + : []; + } + let mapboxKey = "undefined"; + + // see if there's any messages to display to the users + const appMessage = Meteor.settings.public.alert_message + ? Meteor.settings.public.alert_message + : undefined; + + // set meteor settings defaults if they do not exist + if (isEmpty(Meteor.settings.private) || isEmpty(Meteor.settings.public)) { + // create some default meteor settings and write them out + let homeUrl = ""; + if (!process.env.ROOT_URL) { + homeUrl = "https://localhost/home"; + } else { + const homeUrlArr = process.env.ROOT_URL.split("/"); + homeUrlArr.pop(); + homeUrl = `${homeUrlArr.join("/")}/home`; } - // now we have a role roleName - now we need an email - if (existingUserEmail) { - // existing user - no need to verify as the selection list came from the database - // see if it already has the role - authorization = matsCollections.Authorization.findOne({ - email: existingUserEmail, - }); - roles = authorization.roles; - if (roles.indexOf(roleName) === -1) { - // have to add the role - if (roleName) { - roles.push(roleName); + const settings = { + private: { + databases: [], + PYTHON_PATH: "/usr/bin/python3", + MAPBOX_KEY: mapboxKey, + }, + public: { + run_environment: depEnv, + apps_to_score: appsToScore, + default_group: appDefaultGroup, + default_db: appDefaultDB, + default_model: appDefaultModel, + proxy_prefix_path: "", + home: homeUrl, + appName, + mysql_wait_timeout: appTimeOut, + group: appGroup, + app_order: 1, + title: appTitle, + color: appColor, + threshold_units: thresholdUnits, + }, + }; + writeSettings(settings, appName); // this is going to cause the app to restart in the meteor development environment!!! + // exit for production - probably won't ever get here in development mode (running with meteor) + // This depends on whatever system is running the node process to restart it. + console.log("resetApp - exiting after creating default settings"); + process.exit(1); + } + + // mostly for running locally for debugging. We have to be able to choose the app from the app list in deployment.json + // normally (on a server) it will be an environment variable. + // to debug an integration or production deployment, set the environment variable deployment_environment to one of + // development, integration, production, metexpress + if (Meteor.settings.public && Meteor.settings.public.run_environment) { + depEnv = Meteor.settings.public.run_environment; + } else { + depEnv = process.env.NODE_ENV; + } + // get the mapbox key out of the settings file, if it exists + if (Meteor.settings.private && Meteor.settings.private.MAPBOX_KEY) { + mapboxKey = Meteor.settings.private.MAPBOX_KEY; + } + delete Meteor.settings.public.undefinedRoles; + for (let pi = 0; pi < appPools.length; pi += 1) { + const record = appPools[pi]; + const poolName = record.pool; + // if the database credentials have been set in the meteor.private.settings file then the global[poolName] + // will have been defined in the app main.js. Otherwise it will not have been defined. + // If it is undefined (requiring configuration) we will skip it but add + // the corresponding role to Meteor.settings.public.undefinedRoles - + // which will cause the app to route to the configuration page. + if (!global[poolName]) { + console.log(`resetApp adding ${global[poolName]}to undefined roles`); + // There was no pool defined for this poolName - probably needs to be configured so stash the role in the public settings + if (!Meteor.settings.public.undefinedRoles) { + Meteor.settings.public.undefinedRoles = []; + } + Meteor.settings.public.undefinedRoles.push(record.role); + } else { + try { + if (dbType !== matsTypes.DbTypes.couchbase) { + // default to mysql so that old apps won't break + global[poolName].on("connection", function (connection) { + connection.query("set group_concat_max_len = 4294967295"); + connection.query(`set session wait_timeout = ${appTimeOut}`); + // ("opening new " + poolName + " connection"); + }); } - matsCollections.Authorization.upsert( + // connections all work so make sure that Meteor.settings.public.undefinedRoles is undefined + delete Meteor.settings.public.undefinedRoles; + } catch (e) { + console.log( + `${poolName}: not initialized -= 1 could not open connection: Error:${e.message}` + ); + Meteor.settings.public.undefinedRoles = + Meteor.settings.public.undefinedRoles === undefined + ? [] + : Meteor.settings.public.undefinedRoles === undefined; + Meteor.settings.public.undefinedRoles.push(record.role); + } + } + } + // just in case - should never happen. + if ( + Meteor.settings.public.undefinedRoles && + Meteor.settings.public.undefinedRoles.length > 1 + ) { + throw new Meteor.Error( + `dbpools not initialized ${Meteor.settings.public.undefinedRoles}` + ); + } + + // Try getting version from env + let { version: appVersion, commit, branch } = versionInfo.getVersionsFromEnv(); + if (appVersion === "Unknown") { + // Try getting versionInfo from the appProduction database + console.log("VERSION not set in the environment - using localhost"); + appVersion = "localhost"; + commit = "HEAD"; + branch = "feature"; + } + const appType = type || matsTypes.AppTypes.mats; + + // remember that we updated the metadata tables just now - create metaDataTableUpdates + /* + metaDataTableUpdates: { - email: existingUserEmail, + name: dataBaseName, + tables: [tableName1, tableName2 ..], + lastRefreshed : timestamp + } + */ + // only create metadata tables if the resetApp was called with a real metaDataTables object + if (metaDataTableRecords instanceof matsTypes.MetaDataDBRecord) { + const metaDataTables = metaDataTableRecords.getRecords(); + for (let mdti = 0; mdti < metaDataTables.length; mdti += 1) { + const metaDataRef = metaDataTables[mdti]; + metaDataRef.lastRefreshed = moment().format(); + if (metaDataTableUpdates.find({ name: metaDataRef.name }).count() === 0) { + metaDataTableUpdates.update( + { + name: metaDataRef.name, }, + metaDataRef, { - $set: { - roles, - }, + upsert: true, } ); } - } else if (newUserEmail) { - // possible new authorization - see if it happens to exist - authorization = matsCollections.Authorization.findOne({ - email: newUserEmail, + } + } else { + throw new Meteor.Error("Server error: ", "resetApp: bad pool-database entry"); + } + // invoke the standard common routines + matsCollections.Roles.remove({}); + matsDataUtils.doRoles(); + matsCollections.Authorization.remove({}); + matsDataUtils.doAuthorization(); + matsCollections.Credentials.remove({}); + matsDataUtils.doCredentials(); + matsCollections.PlotGraphFunctions.remove({}); + matsCollections.ColorScheme.remove({}); + matsDataUtils.doColorScheme(); + matsCollections.Settings.remove({}); + matsDataUtils.doSettings( + appTitle, + dbType, + appVersion, + commit, + branch, + appName, + appType, + mapboxKey, + appDefaultGroup, + appDefaultDB, + appDefaultModel, + thresholdUnits, + appMessage, + scorecard + ); + matsCollections.PlotParams.remove({}); + matsCollections.CurveTextPatterns.remove({}); + // get the curve params for this app into their collections + matsCollections.CurveParamsInfo.remove({}); + matsCollections.CurveParamsInfo.insert({ + curve_params: curveParams, + }); + for (let cp = 0; cp < curveParams.length; cp += 1) { + if (matsCollections[curveParams[cp]] !== undefined) { + matsCollections[curveParams[cp]].remove({}); + } + } + // if this is a scorecard also get the apps to score out of the settings file + if (Meteor.settings.public && Meteor.settings.public.scorecard) { + if (Meteor.settings.public.apps_to_score) { + appsToScore = Meteor.settings.public.apps_to_score; + matsCollections.AppsToScore.remove({}); + matsCollections.AppsToScore.insert({ + apps_to_score: appsToScore, }); - if (authorization !== undefined) { - // authorization exists - add role to roles if necessary - roles = authorization.roles; - if (roles.indexOf(roleName) === -1) { - // have to add the role - if (roleName) { - roles.push(roleName); - } - matsCollections.Authorization.upsert( - { - email: existingUserEmail, - }, - { - $set: { - roles, - }, - } - ); - } - } else { - // need a new authorization - roles = []; - if (roleName) { - roles.push(roleName); - } - if (newUserEmail) { - matsCollections.Authorization.upsert( - { - email: newUserEmail, - }, - { - $set: { - roles, - }, - } - ); - } - } + } else { + throw new Meteor.Error( + "apps_to_score are not initialized in app settings -= 1cannot build selectors" + ); } - return false; } - }, -}); + // invoke the app specific routines + for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { + await global.appSpecificResetRoutines[ai](); + } + matsCache.clear(); + } +}; -// database controls -const applyDatabaseSettings = new ValidatedMethod({ - name: "matsMethods.applyDatabaseSettings", +const saveLayout = new ValidatedMethod({ + name: "matsMethods.saveLayout", validate: new SimpleSchema({ - settings: { + resultKey: { + type: String, + }, + layout: { + type: Object, + blackbox: true, + }, + curveOpsUpdate: { type: Object, blackbox: true, }, + annotation: { + type: String, + }, }).validator(), - - run(settings) { + run(params) { if (Meteor.isServer) { - if (settings.name) { - matsCollections.Databases.upsert( + const key = params.resultKey; + const { layout } = params; + const { curveOpsUpdate } = params; + const { annotation } = params; + try { + LayoutStoreCollection.upsert( { - name: settings.name, + key, }, { $set: { - name: settings.name, - role: settings.role, - status: settings.status, - host: settings.host, - database: settings.database, - user: settings.user, - password: settings.password, + createdAt: new Date(), + layout, + curveOpsUpdate, + annotation, }, } ); + } catch (error) { + throw new Meteor.Error( + `Error in saveLayout function:${key} : ${error.message}` + ); } - return false; } }, }); -// administration tools -const deleteSettings = new ValidatedMethod({ - name: "matsMethods.deleteSettings", +const saveScorecardSettings = new ValidatedMethod({ + name: "matsMethods.saveScorecardSettings", validate: new SimpleSchema({ - name: { + settingsKey: { + type: String, + }, + scorecardSettings: { type: String, }, }).validator(), run(params) { - if (!Meteor.userId()) { - throw new Meteor.Error("not-logged-in"); - } if (Meteor.isServer) { - matsCollections.CurveSettings.remove({ - name: params.name, - }); + const key = params.settingsKey; + const { scorecardSettings } = params; + try { + // TODO - remove after tests + console.log( + `saveScorecardSettings(${key}):\n${JSON.stringify( + scorecardSettings, + null, + 2 + )}` + ); + // global cbScorecardSettingsPool + (async function (id, doc) { + cbScorecardSettingsPool.upsertCB(id, doc); + })(key, scorecardSettings).then(() => { + console.log("upserted doc with id", key); + }); + // await cbScorecardSettingsPool.upsertCB(settingsKey, scorecardSettings); + } catch (err) { + console.log(`error writing scorecard to database: ${err.message}`); + } } }, }); -// drop a single instance of a scorecard -const dropScorecardInstance = new ValidatedMethod({ - name: "matsMethods.dropScorecardInstance", +// administration tools +const saveSettings = new ValidatedMethod({ + name: "matsMethods.saveSettings", + validate: new SimpleSchema({ + saveAs: { + type: String, + }, + p: { + type: Object, + blackbox: true, + }, + permission: { + type: String, + }, + }).validator(), + run(params) { + const user = "anonymous"; + matsCollections.CurveSettings.upsert( + { + name: params.saveAs, + }, + { + created: moment().format("MM/DD/YYYY HH:mm:ss"), + name: params.saveAs, + data: params.p, + owner: !Meteor.userId() ? "anonymous" : Meteor.userId(), + permission: params.permission, + savedAt: new Date(), + savedBy: !Meteor.user() ? "anonymous" : user, + } + ); + }, +}); + +/* test methods */ + +const testGetMetaDataTableUpdates = new ValidatedMethod({ + name: "matsMethods.testGetMetaDataTableUpdates", + validate: new SimpleSchema({}).validator(), + run() { + return metaDataTableUpdates.find({}).fetch(); + }, +}); + +const testGetTables = new ValidatedMethod({ + name: "matsMethods.testGetTables", validate: new SimpleSchema({ - userName: { + host: { type: String, }, - name: { + port: { type: String, }, - submittedTime: { + user: { type: String, }, - processedAt: { + password: { + type: String, + }, + database: { type: String, }, }).validator(), - run(params) { + async run(params) { if (Meteor.isServer) { - return _dropScorecardInstance( - params.userName, - params.name, - params.submittedTime, - params.processedAt - ); + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + const cbUtilities = new matsCouchbaseUtils.CBUtilities( + params.host, + params.bucket, + params.user, + params.password + ); + try { + const result = await cbUtilities.queryCB("select NOW_MILLIS() as time"); + console.log(`Couchbase get tables suceeded. result: ${result}`); + } catch (err) { + throw new Meteor.Error(err.message); + } + } else { + // default to mysql so that old apps won't break + const Future = require("fibers/future"); + const queryWrap = Future.wrap(function (callback) { + const connection = mysql.createConnection({ + host: params.host, + port: params.port, + user: params.user, + password: params.password, + database: params.database, + }); + connection.query("show tables;", function (err, result) { + if (err || result === undefined) { + // return callback(err,null); + return callback(err, null); + } + const tables = result.map(function (a) { + return a; + }); + + return callback(err, tables); + }); + connection.end(function (err) { + if (err) { + console.log("testGetTables cannot end connection"); + } + }); + }); + try { + return queryWrap().wait(); + } catch (e) { + throw new Meteor.Error(e.message); + } + } } + return null; }, }); -// administration tools -const emailImage = new ValidatedMethod({ - name: "matsMethods.emailImage", - validate: new SimpleSchema({ - imageStr: { - type: String, - }, - toAddress: { - type: String, - }, - subject: { - type: String, - }, - }).validator(), - run(params) { - const { imageStr } = params; - const { toAddress } = params; - const { subject } = params; - if (!Meteor.userId()) { - throw new Meteor.Error(401, "not-logged-in"); - } - const fromAddress = Meteor.user().services.google.email; - // these come from google - see - // http://masashi-k.blogspot.fr/2013/06/sending-mail-with-gmail-using-xoauth2.html - // http://stackoverflow.com/questions/24098461/nodemailer-gmail-what-exactly-is-a-refresh-token-and-how-do-i-get-one/24123550 - - // the gmail account for the credentials is mats.mail.daemon@gmail.com - pwd mats2015! - // var clientId = "339389735380-382sf11aicmgdgn7e72p4end5gnm9sad.apps.googleusercontent.com"; - // var clientSecret = "7CfNN-tRl5QAL595JTW2TkRl"; - // var refresh_token = "1/PDql7FR01N2gmq5NiTfnrT-OlCYC3U67KJYYDNPeGnA"; - const credentials = matsCollections.Credentials.findOne( +const testSetMetaDataTableUpdatesLastRefreshedBack = new ValidatedMethod({ + name: "matsMethods.testSetMetaDataTableUpdatesLastRefreshedBack", + validate: new SimpleSchema({}).validator(), + run() { + const mtu = metaDataTableUpdates.find({}).fetch(); + const id = mtu[0]._id; + metaDataTableUpdates.update( { - name: "oauth_google", + _id: id, }, { - clientId: 1, - clientSecret: 1, - refresh_token: 1, + $set: { + lastRefreshed: 0, + }, } ); - const { clientId } = credentials; - const { clientSecret } = credentials; - const { refresh_token } = credentials; + return metaDataTableUpdates.find({}).fetch(); + }, +}); + +// Define routes for server +if (Meteor.isServer) { + // add indexes to result and axes collections + DownSampleResults.rawCollection().createIndex( + { + createdAt: 1, + }, + { + expireAfterSeconds: 3600 * 8, + } + ); // 8 hour expiration + LayoutStoreCollection.rawCollection().createIndex( + { + createdAt: 1, + }, + { + expireAfterSeconds: 900, + } + ); // 15 min expiration + + // set the default proxy prefix path to "" + // If the settings are not complete, they will be set by the configuration and written out, which will cause the app to reset + if (Meteor.settings.public && !Meteor.settings.public.proxy_prefix_path) { + Meteor.settings.public.proxy_prefix_path = ""; + } + + // eslint-disable-next-line no-unused-vars + Picker.route("/status", function (params, req, res, next) { + Picker.middleware(status(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/status`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(status(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/status`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(status(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getCSV/:key", function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getCSV/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/app:/getCSV/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/CSV/:f/:key/:m/:a", function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/CSV/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/CSV/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getJSON/:key", function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getJSON/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/app:/getJSON/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/JSON/:f/:key/:m/:a", function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/JSON/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/JSON/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/clearCache", function (params, req, res, next) { + Picker.middleware(clearCache(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/clearCache`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(clearCache(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/clearCache`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(clearCache(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getApps", function (params, req, res, next) { + Picker.middleware(getApps(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getApps`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getApps(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getApps`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getApps(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getAppSumsDBs", function (params, req, res, next) { + Picker.middleware(getAppSumsDBs(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getAppSumsDBs`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getAppSumsDBs(res)); + } + ); - let smtpTransporter; - try { - smtpTransporter = Nodemailer.createTransport("SMTP", { - service: "Gmail", - auth: { - XOAuth2: { - user: "mats.gsl@noaa.gov", - clientId, - clientSecret, - refreshToken: refresh_token, - }, - }, - }); - } catch (e) { - throw new Meteor.Error(401, `Transport error ${e.message()}`); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getAppSumsDBs`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getAppSumsDBs(res)); } - try { - const mailOptions = { - sender: fromAddress, - replyTo: fromAddress, - from: fromAddress, - to: toAddress, - subject, - attachments: [ - { - filename: "graph.png", - contents: new Buffer(imageStr.split("base64,")[1], "base64"), - }, - ], - }; + ); - smtpTransporter.sendMail(mailOptions, function (error, response) { - if (error) { - console.log( - `smtpTransporter error ${error} from:${fromAddress} to:${toAddress}` - ); - } else { - console.log(`${response} from:${fromAddress} to:${toAddress}`); - } - smtpTransporter.close(); - }); - } catch (e) { - throw new Meteor.Error(401, `Send error ${e.message()}`); + // eslint-disable-next-line no-unused-vars + Picker.route("/getModels", function (params, req, res, next) { + Picker.middleware(getModels(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getModels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getModels(res)); } - return false; - }, -}); + ); -// administation tool -const getAuthorizations = new ValidatedMethod({ - name: "matsMethods.getAuthorizations", - validate: new SimpleSchema({}).validator(), - run() { - let roles = []; - if (Meteor.isServer) { - const userEmail = Meteor.user().services.google.email.toLowerCase(); - roles = matsCollections.Authorization.findOne({ - email: userEmail, - }).roles; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getModels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getModels(res)); } - return roles; - }, -}); + ); -// administration tool + // eslint-disable-next-line no-unused-vars + Picker.route("/getRegions", function (params, req, res, next) { + Picker.middleware(getRegions(res)); + }); -const getRunEnvironment = new ValidatedMethod({ - name: "matsMethods.getRunEnvironment", - validate: new SimpleSchema({}).validator(), - run() { - return Meteor.settings.public.run_environment; - }, -}); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getRegions`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegions(res)); + } + ); -const getDefaultGroupList = new ValidatedMethod({ - name: "matsMethods.getDefaultGroupList", - validate: new SimpleSchema({}).validator(), - run() { - return matsTypes.DEFAULT_GROUP_LIST; - }, -}); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getRegions`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegions(res)); + } + ); -// retrieves the saved query results (or downsampled results) -const getGraphData = new ValidatedMethod({ - name: "matsMethods.getGraphData", - validate: new SimpleSchema({ - plotParams: { - type: Object, - blackbox: true, - }, - plotType: { - type: String, - }, - expireKey: { - type: Boolean, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const plotGraphFunction = matsCollections.PlotGraphFunctions.findOne({ - plotType: params.plotType, - }); - const { dataFunction } = plotGraphFunction; - let ret; - try { - const hash = require("object-hash"); - const key = hash(params.plotParams); - if (process.env.NODE_ENV === "development" || params.expireKey) { - matsCache.expireKey(key); - } - const results = matsCache.getResult(key); - if (results === undefined) { - // results aren't in the cache - need to process data routine - const Future = require("fibers/future"); - const future = new Future(); - global[dataFunction](params.plotParams, function (results) { - ret = _saveResultData(results); - future.return(ret); - }); - return future.wait(); - } - // results were already in the matsCache (same params and not yet expired) - // are results in the downsampled collection? - const dsResults = DownSampleResults.findOne( - { - key, - }, - {}, - { - disableOplog: true, - } - ); - if (dsResults !== undefined) { - // results are in the mongo cache downsampled collection - returned the downsampled graph data - ret = dsResults; - // update the expire time in the downsampled collection - this requires a new Date - DownSampleResults.rawCollection().update( - { - key, - }, - { - $set: { - createdAt: new Date(), - }, - } - ); - } else { - ret = results; // {key:someKey, result:resultObject} - // refresh expire time. The only way to perform a refresh on matsCache is to re-save the result. - matsCache.storeResult(results.key, results); - } - const sizeof = require("object-sizeof"); - console.log("result.data size is ", sizeof(results)); - return ret; - } catch (dataFunctionError) { - if (dataFunctionError.toLocaleString().indexOf("INFO:") !== -1) { - throw new Meteor.Error(dataFunctionError.message); - } else { - throw new Meteor.Error( - `Error in getGraphData function:${dataFunction} : ${dataFunctionError.message}` - ); - } - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getRegionsValuesMap", function (params, req, res, next) { + Picker.middleware(getRegionsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getRegionsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegionsValuesMap(res)); } - }, -}); + ); -// retrieves the saved query results (or downsampled results) for a specific key -const getGraphDataByKey = new ValidatedMethod({ - name: "matsMethods.getGraphDataByKey", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - let ret; - const key = params.resultKey; - try { - const dsResults = DownSampleResults.findOne( - { - key, - }, - {}, - { - disableOplog: true, - } - ); - if (dsResults !== undefined) { - ret = dsResults; - } else { - ret = matsCache.getResult(key); // {key:someKey, result:resultObject} - } - const sizeof = require("object-sizeof"); - console.log("getGraphDataByKey results size is ", sizeof(dsResults)); - return ret; - } catch (error) { - throw new Meteor.Error( - `Error in getGraphDataByKey function:${key} : ${error.message}` - ); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getRegionsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegionsValuesMap(res)); } - }, -}); + ); -const getLayout = new ValidatedMethod({ - name: "matsMethods.getLayout", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - let ret; - const key = params.resultKey; - try { - ret = LayoutStoreCollection.rawCollection().findOne({ - key, - }); - return ret; - } catch (error) { - throw new Meteor.Error(`Error in getLayout function:${key} : ${error.message}`); - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getStatistics", function (params, req, res, next) { + Picker.middleware(getStatistics(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getStatistics`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatistics(res)); } - }, -}); + ); -const getScorecardSettings = new ValidatedMethod({ - name: "matsMethods.getScorecardSettings", - validate: new SimpleSchema({ - settingsKey: { - type: String, - }, - }).validator(), - async run(params) { - if (Meteor.isServer) { - const key = params.settingsKey; - try { - // global cbScorecardSettingsPool - const rv = await cbScorecardSettingsPool.getCB(key); - return { scorecardSettings: rv.content }; - } catch (error) { - throw new Meteor.Error( - `Error in getScorecardSettings function:${key} : ${error.message}` - ); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getStatistics`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatistics(res)); } - }, -}); + ); -const getPlotParamsFromScorecardInstance = new ValidatedMethod({ - name: "matsMethods.getPlotParamsFromScorecardInstance", - validate: new SimpleSchema({ - userName: { - type: String, - }, - name: { - type: String, - }, - submitted: { - type: String, - }, - processedAt: { - type: String, - }, - }).validator(), - run(params) { - try { - if (Meteor.isServer) { - return _getPlotParamsFromScorecardInstance( - params.userName, - params.name, - params.submitted, - params.processedAt - ); - } - } catch (error) { - throw new Meteor.Error( - `Error in getPlotParamsFromScorecardInstance function:${error.message}` - ); + // eslint-disable-next-line no-unused-vars + Picker.route("/getStatisticsValuesMap", function (params, req, res, next) { + Picker.middleware(getStatisticsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getStatisticsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatisticsValuesMap(res)); } - }, -}); + ); -/* -getPlotResult is used by the graph/text_*_output templates which are used to display textual results. -Because the data isn't being rendered graphically this data is always full size, i.e. NOT downsampled. -That is why it only finds it in the Result file cache, never the DownSampleResult collection. + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getStatisticsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatisticsValuesMap(res)); + } + ); -Because the dataset can be so large ... e.g. megabytes the data retrieval is pagenated. The index is -applied to the underlying datasets.The data gets stripped down and flattened to only contain the data neccesary for text presentation. -A new page index of -1000 means get all the data i.e. no pagenation. - */ -const getPlotResult = new ValidatedMethod({ - name: "matsMethods.getPlotResult", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - pageIndex: { - type: Number, - }, - newPageIndex: { - type: Number, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const rKey = params.resultKey; - const pi = params.pageIndex; - const npi = params.newPageIndex; - let ret = {}; - try { - ret = _getFlattenedResultData(rKey, pi, npi); - } catch (e) { - console.log(e); - } - return ret; + // eslint-disable-next-line no-unused-vars + Picker.route("/getVariables", function (params, req, res, next) { + Picker.middleware(getVariables(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getVariables`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariables(res)); } - }, -}); + ); -const getReleaseNotes = new ValidatedMethod({ - name: "matsMethods.getReleaseNotes", - validate: new SimpleSchema({}).validator(), - run() { - // return Assets.getText('public/MATSReleaseNotes.html'); - // } - if (Meteor.isServer) { - const future = require("fibers/future"); - const fse = require("fs-extra"); - const dFuture = new future(); - let fData; - let file; - if (process.env.NODE_ENV === "development") { - file = `${process.env.PWD}/.meteor/local/build/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; - } else { - file = `${process.env.PWD}/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; - } - try { - fse.readFile(file, "utf8", function (err, data) { - if (err) { - fData = err.message; - dFuture.return(); - } else { - fData = data; - dFuture.return(); - } - }); - } catch (e) { - fData = e.message; - dFuture.return(); - } - dFuture.wait(); - return fData; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getVariables`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariables(res)); } - }, -}); + ); -const setCurveParamDisplayText = new ValidatedMethod({ - name: "matsMethods.setCurveParamDisplayText", - validate: new SimpleSchema({ - paramName: { - type: String, - }, - newText: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - return matsCollections[params.paramName].update( - { name: params.paramName }, - { $set: { controlButtonText: params.newText } } - ); + // eslint-disable-next-line no-unused-vars + Picker.route("/getVariablesValuesMap", function (params, req, res, next) { + Picker.middleware(getVariablesValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getVariablesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariablesValuesMap(res)); } - }, -}); + ); -const getScorecardData = new ValidatedMethod({ - name: "matsMethods.getScorecardData", - validate: new SimpleSchema({ - userName: { - type: String, - }, - name: { - type: String, - }, - submitted: { - type: String, - }, - processedAt: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - return _getScorecardData( - params.userName, - params.name, - params.submitted, - params.processedAt - ); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getVariablesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariablesValuesMap(res)); } - }, -}); + ); -const getScorecardInfo = new ValidatedMethod({ - name: "matsMethods.getScorecardInfo", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - return _getScorecardInfo(); + // eslint-disable-next-line no-unused-vars + Picker.route("/getThresholds", function (params, req, res, next) { + Picker.middleware(getThresholds(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getThresholds`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholds(res)); } - }, -}); + ); -// administration tool -const getUserAddress = new ValidatedMethod({ - name: "matsMethods.getUserAddress", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - return Meteor.user().services.google.email.toLowerCase(); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholds`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholds(res)); } - }, -}); + ); -// app utility -const insertColor = new ValidatedMethod({ - name: "matsMethods.insertColor", - validate: new SimpleSchema({ - newColor: { - type: String, - }, - insertAfterIndex: { - type: Number, - }, - }).validator(), - run(params) { - if (params.newColor === "rgb(255,255,255)") { - return false; + // eslint-disable-next-line no-unused-vars + Picker.route("/getThresholdsValuesMap", function (params, req, res, next) { + Picker.middleware(getThresholdsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getThresholdsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholdsValuesMap(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholdsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholdsValuesMap(res)); } - const colorScheme = matsCollections.ColorScheme.findOne({}); - colorScheme.colors.splice(params.insertAfterIndex, 0, newColor); - matsCollections.update({}, colorScheme); - return false; - }, -}); + ); -// administration tool -const readFunctionFile = new ValidatedMethod({ - name: "matsMethods.readFunctionFile", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - const future = require("fibers/future"); - const fse = require("fs-extra"); - let path = ""; - let fData; - if (type === "data") { - path = `/web/static/dataFunctions/${file}`; - console.log(`exporting data file: ${path}`); - } else if (type === "graph") { - path = `/web/static/displayFunctions/${file}`; - console.log(`exporting graph file: ${path}`); - } else { - return "error - wrong type"; - } - fse.readFile(path, function (err, data) { - if (err) throw err; - fData = data.toString(); - future.return(fData); - }); - return future.wait(); + // eslint-disable-next-line no-unused-vars + Picker.route("/getScales", function (params, req, res, next) { + Picker.middleware(getScales(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getScales`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScales(res)); } - }, -}); + ); -// refreshes the metadata for the app that's running -const refreshMetaData = new ValidatedMethod({ - name: "matsMethods.refreshMetaData", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - try { - // console.log("GUI asked to refresh metadata"); - _checkMetaDataRefresh(); - } catch (e) { - console.log(e); - throw new Meteor.Error("Server error: ", e.message); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getScales`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScales(res)); } - return metaDataTableUpdates.find({}).fetch(); - }, -}); + ); -// administation tool -const removeAuthorization = new ValidatedMethod({ - name: "matsMethods.removeAuthorization", - validate: new SimpleSchema({ - settings: { - type: Object, - blackbox: true, - }, - }).validator(), - run(settings) { - if (Meteor.isServer) { - let email; - let roleName; - const { userRoleName } = settings; - const { authorizationRole } = settings; - const { newUserEmail } = settings; - const { existingUserEmail } = settings; - if (authorizationRole) { - // existing role - the role roleName - no need to verify as the selection list came from the database - roleName = authorizationRole; - } else if (userRoleName) { - roleName = userRoleName; - } - if (existingUserEmail) { - email = existingUserEmail; - } else { - email = newUserEmail; - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getScalesValuesMap", function (params, req, res, next) { + Picker.middleware(getScalesValuesMap(res)); + }); - // if user and role remove the role from the user - if (email && roleName) { - matsCollections.Authorization.update( - { - email, - }, - { - $pull: { - roles: roleName, - }, - } - ); - } - // if user and no role remove the user - if (email && !roleName) { - matsCollections.Authorization.remove({ - email, - }); - } - // if role and no user remove role and remove role from all users - if (roleName && !email) { - // remove the role - matsCollections.Roles.remove({ - name: roleName, - }); - // remove the roleName role from all the authorizations - matsCollections.Authorization.update( - { - roles: roleName, - }, - { - $pull: { - roles: roleName, - }, - }, - { - multi: true, - } - ); - } - return false; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getScalesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScalesValuesMap(res)); } - }, -}); - -// app utility -const removeColor = new ValidatedMethod({ - name: "matsMethods.removeColor", - validate: new SimpleSchema({ - removeColor: { - type: String, - }, - }).validator(), - run(removeColor) { - const colorScheme = matsCollections.ColorScheme.findOne({}); - const removeIndex = colorScheme.colors.indexOf(removeColor); - colorScheme.colors.splice(removeIndex, 1); - matsCollections.ColorScheme.update({}, colorScheme); - return false; - }, -}); + ); -// database controls -const removeDatabase = new ValidatedMethod({ - name: "matsMethods.removeDatabase", - validate: new SimpleSchema({ - dbName: { - type: String, - }, - }).validator(), - run(dbName) { - if (Meteor.isServer) { - matsCollections.Databases.remove({ - name: dbName, - }); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getScalesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScalesValuesMap(res)); } - }, -}); + ); -const applySettingsData = new ValidatedMethod({ - name: "matsMethods.applySettingsData", - validate: new SimpleSchema({ - settings: { - type: Object, - blackbox: true, - }, - }).validator(), - // this method forces a restart on purpose. We do not want retries - applyOptions: { - noRetry: true, - }, - run(settingsParam) { - if (Meteor.isServer) { - // Read the existing settings file - const { settings } = settingsParam; - console.log( - "applySettingsData - matsCollections.appName.findOne({}) is ", - matsCollections.appName.findOne({}) - ); - const { appName } = matsCollections.Settings.findOne({}); - _write_settings(settings, appName); - // in development - when being run by meteor, this should force a restart of the app. - // in case I am in a container - exit and force a reload - console.log( - `applySettingsData - process.env.NODE_ENV is: ${process.env.NODE_ENV}` - ); - if (process.env.NODE_ENV !== "development") { - console.log("applySettingsData - exiting after writing new Settings"); - process.exit(0); - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getTruths", function (params, req, res, next) { + Picker.middleware(getTruths(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getTruths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruths(res)); } - }, -}); + ); -// makes sure all of the parameters display appropriate selections in relation to one another -// for default settings ... -const resetApp = async function (appRef) { - if (Meteor.isServer) { - const fse = require("fs-extra"); - const metaDataTableRecords = appRef.appMdr; - const { appPools } = appRef; - const type = appRef.appType; - const scorecard = Meteor.settings.public.scorecard - ? Meteor.settings.public.scorecard - : false; - const dbType = appRef.dbType ? appRef.dbType : matsTypes.DbTypes.mysql; - const appName = Meteor.settings.public.app ? Meteor.settings.public.app : "unnamed"; - const appTitle = Meteor.settings.public.title - ? Meteor.settings.public.title - : "Unnamed App"; - const appGroup = Meteor.settings.public.group - ? Meteor.settings.public.group - : "Misc. Apps"; - const thresholdUnits = Meteor.settings.public.threshold_units - ? Meteor.settings.public.threshold_units - : {}; - let appDefaultGroup = ""; - let appDefaultDB = ""; - let appDefaultModel = ""; - let appColor; - switch (type) { - case matsTypes.AppTypes.mats: - if (dbType === matsTypes.DbTypes.couchbase) { - appColor = "#33abbb"; - } else { - appColor = Meteor.settings.public.color - ? Meteor.settings.public.color - : "#3366bb"; - } - break; - case matsTypes.AppTypes.metexpress: - appColor = Meteor.settings.public.color - ? Meteor.settings.public.color - : "darkorchid"; - appDefaultGroup = Meteor.settings.public.default_group - ? Meteor.settings.public.default_group - : "NO GROUP"; - appDefaultDB = Meteor.settings.public.default_db - ? Meteor.settings.public.default_db - : "mv_default"; - appDefaultModel = Meteor.settings.public.default_model - ? Meteor.settings.public.default_model - : "Default"; - break; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getTruths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruths(res)); } - const appTimeOut = Meteor.settings.public.mysql_wait_timeout - ? Meteor.settings.public.mysql_wait_timeout - : 300; - let dep_env = process.env.NODE_ENV; - const curve_params = curveParamsByApp[Meteor.settings.public.app]; - let apps_to_score; - if (Meteor.settings.public.scorecard) { - apps_to_score = Meteor.settings.public.apps_to_score - ? Meteor.settings.public.apps_to_score - : []; + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getTruthsValuesMap", function (params, req, res, next) { + Picker.middleware(getTruthsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getTruthsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruthsValuesMap(res)); } - let mapboxKey = "undefined"; + ); - // see if there's any messages to display to the users - const appMessage = Meteor.settings.public.alert_message - ? Meteor.settings.public.alert_message - : undefined; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getTruthsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruthsValuesMap(res)); + } + ); - // set meteor settings defaults if they do not exist - if (isEmpty(Meteor.settings.private) || isEmpty(Meteor.settings.public)) { - // create some default meteor settings and write them out - let homeUrl = ""; - if (!process.env.ROOT_URL) { - homeUrl = "https://localhost/home"; - } else { - const homeUrlArr = process.env.ROOT_URL.split("/"); - homeUrlArr.pop(); - homeUrl = `${homeUrlArr.join("/")}/home`; - } - const settings = { - private: { - databases: [], - PYTHON_PATH: "/usr/bin/python3", - MAPBOX_KEY: mapboxKey, - }, - public: { - run_environment: dep_env, - apps_to_score, - default_group: appDefaultGroup, - default_db: appDefaultDB, - default_model: appDefaultModel, - proxy_prefix_path: "", - home: homeUrl, - appName, - mysql_wait_timeout: appTimeOut, - group: appGroup, - app_order: 1, - title: appTitle, - color: appColor, - threshold_units: thresholdUnits, - }, - }; - _write_settings(settings, appName); // this is going to cause the app to restart in the meteor development environment!!! - // exit for production - probably won't ever get here in development mode (running with meteor) - // This depends on whatever system is running the node process to restart it. - console.log("resetApp - exiting after creating default settings"); - process.exit(1); + // eslint-disable-next-line no-unused-vars + Picker.route("/getFcstLengths", function (params, req, res, next) { + Picker.middleware(getFcstLengths(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getFcstLengths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstLengths(res)); } + ); - // mostly for running locally for debugging. We have to be able to choose the app from the app list in deployment.json - // normally (on a server) it will be an environment variable. - // to debug an integration or production deployment, set the environment variable deployment_environment to one of - // development, integration, production, metexpress - if (Meteor.settings.public && Meteor.settings.public.run_environment) { - dep_env = Meteor.settings.public.run_environment; - } else { - dep_env = process.env.NODE_ENV; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstLengths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstLengths(res)); } - // get the mapbox key out of the settings file, if it exists - if (Meteor.settings.private && Meteor.settings.private.MAPBOX_KEY) { - mapboxKey = Meteor.settings.private.MAPBOX_KEY; + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getFcstTypes", function (params, req, res, next) { + Picker.middleware(getFcstTypes(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getFcstTypes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypes(res)); } - delete Meteor.settings.public.undefinedRoles; - for (let pi = 0; pi < appPools.length; pi++) { - const record = appPools[pi]; - const poolName = record.pool; - // if the database credentials have been set in the meteor.private.settings file then the global[poolName] - // will have been defined in the app main.js. Otherwise it will not have been defined. - // If it is undefined (requiring configuration) we will skip it but add - // the corresponding role to Meteor.settings.public.undefinedRoles - - // which will cause the app to route to the configuration page. - if (!global[poolName]) { - console.log(`resetApp adding ${global[poolName]}to undefined roles`); - // There was no pool defined for this poolName - probably needs to be configured so stash the role in the public settings - if (!Meteor.settings.public.undefinedRoles) { - Meteor.settings.public.undefinedRoles = []; - } - Meteor.settings.public.undefinedRoles.push(record.role); - continue; - } - try { - if (dbType === matsTypes.DbTypes.couchbase) { - // simple couchbase test - const time = await cbPool.queryCB("select NOW_MILLIS() as time;"); - break; - } else { - // default to mysql so that old apps won't break - global[poolName].on("connection", function (connection) { - connection.query("set group_concat_max_len = 4294967295"); - connection.query(`set session wait_timeout = ${appTimeOut}`); - // ("opening new " + poolName + " connection"); - }); - } - } catch (e) { - console.log( - `${poolName}: not initialized-- could not open connection: Error:${e.message}` - ); - Meteor.settings.public.undefinedRoles = - Meteor.settings.public.undefinedRoles === undefined - ? [] - : Meteor.settings.public.undefinedRoles === undefined; - Meteor.settings.public.undefinedRoles.push(record.role); - continue; - } - // connections all work so make sure that Meteor.settings.public.undefinedRoles is undefined - delete Meteor.settings.public.undefinedRoles; + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypes(res)); } - // just in case - should never happen. - if ( - Meteor.settings.public.undefinedRoles && - Meteor.settings.public.undefinedRoles.length > 1 - ) { - throw new Meteor.Error( - `dbpools not initialized ${Meteor.settings.public.undefinedRoles}` - ); + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getFcstTypesValuesMap", function (params, req, res, next) { + Picker.middleware(getFcstTypesValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getFcstTypesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypesValuesMap(res)); } + ); - // Try getting version from env - let { version: appVersion, commit, branch } = versionInfo.getVersionsFromEnv(); - if (appVersion === "Unknown") { - // Try getting versionInfo from the appProduction database - console.log("VERSION not set in the environment - using localhost"); - appVersion = "localhost"; - commit = "HEAD"; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypesValuesMap(res)); } - const appType = type || matsTypes.AppTypes.mats; + ); - // remember that we updated the metadata tables just now - create metaDataTableUpdates - /* - metaDataTableUpdates: - { - name: dataBaseName, - tables: [tableName1, tableName2 ..], - lastRefreshed : timestamp - } - */ - // only create metadata tables if the resetApp was called with a real metaDataTables object - if (metaDataTableRecords instanceof matsTypes.MetaDataDBRecord) { - const metaDataTables = metaDataTableRecords.getRecords(); - for (let mdti = 0; mdti < metaDataTables.length; mdti++) { - const metaDataRef = metaDataTables[mdti]; - metaDataRef.lastRefreshed = moment().format(); - if (metaDataTableUpdates.find({ name: metaDataRef.name }).count() === 0) { - metaDataTableUpdates.update( - { - name: metaDataRef.name, - }, - metaDataRef, - { - upsert: true, - } - ); - } - } - } else { - throw new Meteor.Error("Server error: ", "resetApp: bad pool-database entry"); + // eslint-disable-next-line no-unused-vars + Picker.route("/getValidTimes", function (params, req, res, next) { + Picker.middleware(getValidTimes(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getValidTimes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getValidTimes(res)); } - // invoke the standard common routines - matsCollections.Roles.remove({}); - matsDataUtils.doRoles(); - matsCollections.Authorization.remove({}); - matsDataUtils.doAuthorization(); - matsCollections.Credentials.remove({}); - matsDataUtils.doCredentials(); - matsCollections.PlotGraphFunctions.remove({}); - matsCollections.ColorScheme.remove({}); - matsDataUtils.doColorScheme(); - matsCollections.Settings.remove({}); - matsDataUtils.doSettings( - appTitle, - dbType, - appVersion, - commit, - appName, - appType, - mapboxKey, - appDefaultGroup, - appDefaultDB, - appDefaultModel, - thresholdUnits, - appMessage, - scorecard - ); - matsCollections.PlotParams.remove({}); - matsCollections.CurveTextPatterns.remove({}); - // get the curve params for this app into their collections - matsCollections.CurveParamsInfo.remove({}); - matsCollections.CurveParamsInfo.insert({ - curve_params, - }); - for (let cp = 0; cp < curve_params.length; cp++) { - if (matsCollections[curve_params[cp]] !== undefined) { - matsCollections[curve_params[cp]].remove({}); - } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getValidTimes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getValidTimes(res)); } - // if this is a scorecard also get the apps to score out of the settings file - if (Meteor.settings.public && Meteor.settings.public.scorecard) { - if (Meteor.settings.public.apps_to_score) { - apps_to_score = Meteor.settings.public.apps_to_score; - matsCollections.AppsToScore.remove({}); - matsCollections.AppsToScore.insert({ - apps_to_score, - }); - } else { - throw new Meteor.Error( - "apps_to_score are not initialized in app settings--cannot build selectors" - ); - } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getLevels", function (params, req, res, next) { + Picker.middleware(getLevels(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getLevels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getLevels(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getLevels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getLevels(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getDates", function (params, req, res, next) { + Picker.middleware(getDates(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getDates`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getDates(res)); } - // invoke the app specific routines - for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { - await global.appSpecificResetRoutines[ai](); + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getDates`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getDates(res)); } - matsCache.clear(); - } -}; + ); -const saveLayout = new ValidatedMethod({ - name: "matsMethods.saveLayout", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - layout: { - type: Object, - blackbox: true, - }, - curveOpsUpdate: { - type: Object, - blackbox: true, - }, - annotation: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const key = params.resultKey; - const { layout } = params; - const { curveOpsUpdate } = params; - const { annotation } = params; - try { - LayoutStoreCollection.upsert( - { - key, - }, - { - $set: { - createdAt: new Date(), - layout, - curveOpsUpdate, - annotation, - }, - } - ); - } catch (error) { - throw new Meteor.Error( - `Error in saveLayout function:${key} : ${error.message}` - ); - } + // create picker routes for refreshMetaData + // eslint-disable-next-line no-unused-vars + Picker.route("/refreshMetadata", function (params, req, res, next) { + Picker.middleware(refreshMetadataMWltData(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/refreshMetadata`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshMetadataMWltData(res)); } - }, -}); + ); -const saveScorecardSettings = new ValidatedMethod({ - name: "matsMethods.saveScorecardSettings", - validate: new SimpleSchema({ - settingsKey: { - type: String, - }, - scorecardSettings: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const key = params.settingsKey; - const { scorecardSettings } = params; - try { - // TODO - remove after tests - console.log( - `saveScorecardSettings(${key}):\n${JSON.stringify( - scorecardSettings, - null, - 2 - )}` - ); - // global cbScorecardSettingsPool - (async function (id, doc) { - cbScorecardSettingsPool.upsertCB(id, doc); - })(key, scorecardSettings).then(() => { - console.log("upserted doc with id", key); - }); - // await cbScorecardSettingsPool.upsertCB(settingsKey, scorecardSettings); - } catch (err) { - console.log(`error writing scorecard to database: ${err.message}`); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/refreshMetadata`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshMetadataMWltData(res)); } - }, -}); + ); + // eslint-disable-next-line no-unused-vars + Picker.route("/refreshScorecard/:docId", function (params, req, res, next) { + Picker.middleware(refreshScorecard(params, res)); + }); -// administration tools -const saveSettings = new ValidatedMethod({ - name: "matsMethods.saveSettings", - validate: new SimpleSchema({ - saveAs: { - type: String, - }, - p: { - type: Object, - blackbox: true, - }, - permission: { - type: String, - }, - }).validator(), - run(params) { - const user = "anonymous"; - matsCollections.CurveSettings.upsert( - { - name: params.saveAs, - }, - { - created: moment().format("MM/DD/YYYY HH:mm:ss"), - name: params.saveAs, - data: params.p, - owner: !Meteor.userId() ? "anonymous" : Meteor.userId(), - permission: params.permission, - savedAt: new Date(), - savedBy: !Meteor.user() ? "anonymous" : user, - } - ); - }, -}); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/refreshScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshScorecard(params, res)); + } + ); -/* test methods */ + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/refreshScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshScorecard(params, res)); + } + ); -const testGetMetaDataTableUpdates = new ValidatedMethod({ - name: "matsMethods.testGetMetaDataTableUpdates", - validate: new SimpleSchema({}).validator(), - run() { - return metaDataTableUpdates.find({}).fetch(); - }, -}); + // eslint-disable-next-line no-unused-vars + Picker.route("/setStatusScorecard/:docId", function (params, req, res, next) { + Picker.middleware(setStatusScorecard(params, req, res)); + }); -const testGetTables = new ValidatedMethod({ - name: "matsMethods.testGetTables", - validate: new SimpleSchema({ - host: { - type: String, - }, - port: { - type: String, - }, - user: { - type: String, - }, - password: { - type: String, - }, - database: { - type: String, - }, - }).validator(), - async run(params) { - if (Meteor.isServer) { - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - const cbUtilities = new matsCouchbaseUtils.CBUtilities( - params.host, - params.bucket, - params.user, - params.password - ); - try { - const result = await cbUtilities.queryCB("select NOW_MILLIS() as time"); - } catch (err) { - throw new Meteor.Error(e.message); - } - } else { - // default to mysql so that old apps won't break - const Future = require("fibers/future"); - const queryWrap = Future.wrap(function (callback) { - const connection = mysql.createConnection({ - host: params.host, - port: params.port, - user: params.user, - password: params.password, - database: params.database, - }); - connection.query("show tables;", function (err, result) { - if (err || result === undefined) { - // return callback(err,null); - return callback(err, null); - } - const tables = result.map(function (a) { - return a; - }); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/setStatusScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(setStatusScorecard(params, req, res)); + } + ); - return callback(err, tables); - }); - connection.end(function (err) { - if (err) { - console.log("testGetTables cannot end connection"); - } - }); - }); - try { - return queryWrap().wait(); - } catch (e) { - throw new Meteor.Error(e.message); - } - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/setStatusScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(setStatusScorecard(params, req, res)); } - }, -}); + ); +} -const testSetMetaDataTableUpdatesLastRefreshedBack = new ValidatedMethod({ - name: "matsMethods.testSetMetaDataTableUpdatesLastRefreshedBack", - validate: new SimpleSchema({}).validator(), - run() { - const mtu = metaDataTableUpdates.find({}).fetch(); - const id = mtu[0]._id; - metaDataTableUpdates.update( - { - _id: id, - }, - { - $set: { - lastRefreshed: 0, - }, - } - ); - return metaDataTableUpdates.find({}).fetch(); - }, -}); +// eslint-disable-next-line no-undef export default matsMethods = { addSentAddress, applyAuthorization, diff --git a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js index fbbf1917d..4ef5dde68 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js @@ -2,7 +2,7 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsCollections, matsTypes } from "meteor/randyp:mats-common"; +import { matsCollections, matsTypes, matsDataUtils } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; import { moment } from "meteor/momentjs:moment"; import { _ } from "meteor/underscore"; @@ -1497,7 +1497,7 @@ const generateGridScaleProbPlotOptions = function (axisMap) { const yPad = (ymax - ymin) * 0.025 !== 0 ? (ymax - ymin) * 0.025 : 0.025; const newYmax = Math.log10(ymax + yPad * 100); const newYmin = - Number.isNaN(Math.log10(ymin - yPad)) || Math.log10(ymin - yPad) < 1 + matsDataUtils.isThisANaN(Math.log10(ymin - yPad)) || Math.log10(ymin - yPad) < 1 ? 0 : Math.log10(ymin - yPad); layout.yaxis.range = [newYmin, newYmax]; diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index 13906a9f0..7382394f7 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -125,11 +125,11 @@ const processDataXYCurve = function ( diffFrom === null || !( Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[0]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[1]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_y.array[di] = null; @@ -206,46 +206,46 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.y[di] - errorLength).toPrecision(4)} to ${Number( @@ -255,27 +255,27 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.y[di], nGood: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; @@ -656,11 +656,11 @@ const processDataProfile = function ( diffFrom === null || !( Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[0]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[1]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_x.array[di] = null; @@ -696,23 +696,23 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; @@ -721,23 +721,23 @@ const processDataProfile = function ( `
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.x[di] - errorLength).toPrecision( @@ -747,27 +747,27 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.x[di], nGood: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index 29a955763..061270e15 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -391,7 +391,7 @@ const parseQueryDataXYCurve = function ( const n = rows[rowIndex].sub_data.toString().split(",").length; if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -417,7 +417,7 @@ const parseQueryDataXYCurve = function ( absSum, statisticStr ); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -470,7 +470,7 @@ const parseQueryDataXYCurve = function ( if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -508,7 +508,7 @@ const parseQueryDataXYCurve = function ( } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -552,7 +552,7 @@ const parseQueryDataXYCurve = function ( } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1024,7 +1024,7 @@ const parseQueryDataReliability = function (rows, d, appParams, kernel) { currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1194,7 +1194,7 @@ const parseQueryDataPerformanceDiagram = function (rows, d, appParams) { currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1379,7 +1379,7 @@ const parseQueryDataSimpleScatter = function ( absSumX, statisticXStr ); - xStat = Number.isNaN(Number(xStat)) ? null : xStat; + xStat = matsDataUtils.isThisANaN(Number(xStat)) ? null : xStat; yStat = matsDataUtils.calculateStatScalar( squareDiffSumY, NSumY, @@ -1389,7 +1389,7 @@ const parseQueryDataSimpleScatter = function ( absSumY, statisticYStr ); - yStat = Number.isNaN(Number(yStat)) ? null : yStat; + yStat = matsDataUtils.isThisANaN(Number(yStat)) ? null : yStat; } else { xStat = null; yStat = null; @@ -1429,7 +1429,7 @@ const parseQueryDataSimpleScatter = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1722,7 +1722,7 @@ const parseQueryDataMapScalar = function ( absSum, `${statistic}_${variable}` ); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; } else { queryVal = null; } @@ -1752,7 +1752,7 @@ const parseQueryDataMapScalar = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1830,7 +1830,7 @@ const parseQueryDataMapScalar = function ( absSumSum, `${statistic}_${variable}` ); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database e.message = `Error in parseQueryDataMapScalar. The expected fields don't seem to be present in the results cache: ${e.message}`; @@ -2007,7 +2007,7 @@ const parseQueryDataMapCTC = function ( const n = rows[rowIndex].nTimes; if (hit + fa + miss + cn > 0) { queryVal = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statistic); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; switch (statistic) { case "PODy (POD of value < threshold)": case "PODy (POD of value > threshold)": @@ -2068,7 +2068,7 @@ const parseQueryDataMapCTC = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2135,7 +2135,7 @@ const parseQueryDataMapCTC = function ( thisSubHit.length, statistic ); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database e.message = `Error in parseQueryDataMapCTC. The expected fields don't seem to be present in the results cache: ${e.message}`; @@ -2325,7 +2325,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { const n = rows[rowIndex].sub_data.toString().split(",").length; if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -2351,7 +2351,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { absSum, statisticStr ); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -2376,7 +2376,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2406,7 +2406,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2438,7 +2438,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2590,7 +2590,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { cn = Number(rows[rowIndex].cn); if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } } else if ( rows[rowIndex].stat === undefined && @@ -2614,7 +2614,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { absSum, statisticStr ); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; const variable = statisticStr.split("_")[1]; stdev = matsDataUtils.calculateStatScalar( squareDiffSum, @@ -2667,7 +2667,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2685,7 +2685,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2707,7 +2707,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index 9500a50a9..31a63e32f 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -9,6 +9,12 @@ import { HTTP } from "meteor/jkuester:http"; /* eslint-disable global-require */ /* eslint-disable no-console */ +// wrapper for NaN check +const isThisANaN = function (val) { + // eslint-disable-next-line no-restricted-globals + return !val || isNaN(val); +}; + // this function checks if two JSON objects are identical const areObjectsEqual = function (o, p) { if ((o && !p) || (p && !o)) { @@ -348,6 +354,7 @@ const doSettings = function ( dbType, version, commit, + branch, appName, appType, mapboxKey, @@ -458,7 +465,7 @@ const callMetadataAPI = function ( // calculates the statistic for ctc plots const calculateStatCTC = function (hit, fa, miss, cn, n, statistic) { - if (Number.isNaN(hit) || Number.isNaN(fa) || Number.isNaN(miss) || Number.isNaN(cn)) + if (isThisANaN(hit) || isThisANaN(fa) || isThisANaN(miss) || isThisANaN(cn)) return null; let queryVal; switch (statistic) { @@ -540,12 +547,12 @@ const calculateStatScalar = function ( statisticAndVariable ) { if ( - Number.isNaN(squareDiffSum) || - Number.isNaN(NSum) || - Number.isNaN(obsModelDiffSum) || - Number.isNaN(modelSum) || - Number.isNaN(obsSum) || - Number.isNaN(absSum) + isThisANaN(squareDiffSum) || + isThisANaN(NSum) || + isThisANaN(obsModelDiffSum) || + isThisANaN(modelSum) || + isThisANaN(obsSum) || + isThisANaN(absSum) ) return null; let queryVal; @@ -578,7 +585,7 @@ const calculateStatScalar = function ( default: queryVal = null; } - if (Number.isNaN(queryVal)) return null; + if (isThisANaN(queryVal)) return null; // need to convert to correct units for surface data but not upperair if (statistic !== "N") { if ( @@ -1191,7 +1198,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { let secs; let delta; for (i = 0; i < sSecs.length; i += 1) { - if (Number.isNaN(sVals[i])) { + if (isThisANaN(sVals[i])) { n -= 1; } else { secs = sSecs[i]; @@ -1214,7 +1221,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { console.log(`matsDataUtil.getErr: ${error}`); } for (i = 0; i < sVals.length; i += 1) { - if (!Number.isNaN(sVals[i])) { + if (!isThisANaN(sVals[i])) { minVal = minVal < sVals[i] ? minVal : sVals[i]; maxVal = maxVal > sVals[i] ? maxVal : sVals[i]; dataSum += sVals[i]; @@ -1247,7 +1254,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { let nDeltas = 0; for (i = 0; i < sSecs.length; i += 1) { - if (!Number.isNaN(sVals[i])) { + if (!isThisANaN(sVals[i])) { let sec = sSecs[i]; if (typeof sec === "string" || sec instanceof String) sec = Number(sec); let lev; @@ -1544,7 +1551,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins": // get the user's chosen number of bins binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1559,7 +1566,7 @@ const setHistogramParameters = function (plotParams) { case "Choose a bin bound": // let the histogram routine know that we want the bins shifted over to whatever was input pivotVal = Number(plotParams["bin-pivot"]); - if (Number.isNaN(pivotVal)) { + if (isThisANaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1567,7 +1574,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and make zero a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to zero binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1578,13 +1585,13 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and choose a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to whatever was input binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } pivotVal = Number(plotParams["bin-pivot"]); - if (Number.isNaN(pivotVal)) { + if (isThisANaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1595,7 +1602,7 @@ const setHistogramParameters = function (plotParams) { binBounds = plotParams["bin-bounds"].split(",").map(function (item) { let thisItem = item.trim(); thisItem = Number(thisItem); - if (!Number.isNaN(thisItem)) { + if (!isThisANaN(thisItem)) { return thisItem; } throw new Error( @@ -1619,17 +1626,17 @@ const setHistogramParameters = function (plotParams) { case "Manual bin start, number, and stride": // get the bin start, number, and stride. binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } binStart = Number(plotParams["bin-start"]); - if (Number.isNaN(binStart)) { + if (isThisANaN(binStart)) { throw new Error("Error parsing bin start: please enter the desired bin start."); } binStride = Number(plotParams["bin-stride"]); - if (Number.isNaN(binStride)) { + if (isThisANaN(binStride)) { throw new Error( "Error parsing bin stride: please enter the desired bin stride." ); @@ -1708,7 +1715,7 @@ const calculateHistogramBins = function ( binLowBounds[binParams.binNum - 1] = fullUpBound; binMeans[binParams.binNum - 1] = fullUpBound + binInterval / 2; - if (binParams.pivotVal !== undefined && !Number.isNaN(binParams.pivotVal)) { + if (binParams.pivotVal !== undefined && !isThisANaN(binParams.pivotVal)) { // need to shift the bounds and means over so that one of the bounds is on the chosen pivot const closestBoundToPivot = binLowBounds.reduce(function (prev, curr) { return Math.abs(curr - binParams.pivotVal) < Math.abs(prev - binParams.pivotVal) @@ -2180,6 +2187,7 @@ export default matsDataUtils = { average, median, stdev, + isThisANaN, dateConvert, getDateRange, secsConvert, diff --git a/meteor_packages/mats-common/package.js b/meteor_packages/mats-common/package.js index 15ebd7bb5..b872f5da8 100644 --- a/meteor_packages/mats-common/package.js +++ b/meteor_packages/mats-common/package.js @@ -30,10 +30,10 @@ Package.onUse(function (api) { "node-file-cache": "1.0.2", "python-shell": "5.0.0", couchbase: "4.3.1", - "simpl-schema": "3.4.6", "vanillajs-datepicker": "1.3.4", daterangepicker: "3.1.0", "lighten-darken-color": "1.0.0", + nodemailer: "6.9.14", }); api.mainModule("server/main.js", "server"); api.mainModule("client/main.js", "client"); @@ -67,6 +67,8 @@ Package.onUse(function (api) { api.use("momentjs:moment"); api.use("pcel:mysql"); api.use("reactive-var"); + api.use("jkuester:http"); + api.use("aldeed:simple-schema"); // modules api.export("matsCollections", ["client", "server"]); @@ -346,51 +348,3 @@ Package.onUse(function (api) { "server" ); }); - -Package.onTest(function (api) { - api.use("ecmascript"); - api.use("meteortesting:mocha"); - api.use("randyp:mats-common"); - api.addFiles("imports/startup/api/version-info-tests.js"); - - // try duplicating the runtime deps - Npm.depends({ - "fs-extra": "7.0.0", - "@babel/runtime": "7.10.4", - "meteor-node-stubs": "0.4.1", - url: "0.11.0", - "jquery-ui": "1.12.1", - "csv-stringify": "4.3.1", - "node-file-cache": "1.0.2", - "python-shell": "3.0.1", - couchbase: "3.2.3", - "simpl-schema": "1.12.0", - }); - api.use("natestrauser:select2", "client"); - api.use("mdg:validated-method"); - api.use("ecmascript"); - api.use("modules"); - api.imply("ecmascript"); - api.use(["templating"], "client"); - api.use("accounts-ui", "client"); - api.use("accounts-password", "client"); - api.use("service-configuration", "server"); - api.use("yasinuslu:json-view", "client"); - api.use("mdg:validated-method"); - api.use("session"); - api.imply("session"); - api.use("twbs:bootstrap"); - api.use("msavin:mongol"); - api.use("differential:event-hooks"); - api.use("risul:bootstrap-colorpicker"); - api.use("logging"); - api.use("reload"); - api.use("random"); - api.use("ejson"); - api.use("spacebars"); - api.use("check"); - api.use("ostrio:flow-router-extra"); - api.use("meteorhacks:picker"); - api.use("momentjs:moment"); - api.use("pcel:mysql"); -});