From 0851edb0ea72e0b538b94a63808c8b9ab1d6ef5d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 07:46:42 -0600 Subject: [PATCH] Update develop-ref after #2610 (#2613) * Bugfix #2244 develop fix diff tests (#2254) * skip diff for specific png images that produce diffs occasionally but look the same visually * add 2 more keywords for use cases that occasionally flag small, unnoticeable diffs in png images * Feature #2253 update tests for diff_util (#2256) Co-authored-by: root * Feature #1974 Documentation: update coding standards section (#2260) * Feature #2253 conftest fixture to set pytest tmpdir (#2261) * Feature #2151 Documentation: Add quick search keywords METplotpy/calcpy use cases (#2259) * Feature #2138 CI enhance MET Docker tag override (#2258) * Updating development instructions. * fixed typo * fixed another typo * Feature #2207 Documentation - Updates to Contributor's Guide (#2263) Co-authored-by: Dan Adriaansen * Feature develop update install files (#2266) * Updating installation files * Removing run_commands.sh * Updated the file to add a MINICONDA_PATH * Adding files for jet * Adding orion file * Removing beta references * Corrected met version * Adding files for WCOSS2 machines * Removing rc1 from acorn file * Removing beta1 * Bugfix #1853 develop - PointStat don't require mask variables to be set (#2262) * update version of certifi to match auto-generated dependabot PR #2267 * feature #2253 more tests for diff_util.py (#2264) * update versions of METplus components for the next official release * updated release notes for 5.1.0 release * removed duplicate entries in release notes * Added EC2 instance recipee with S3 mounting ability (#2269) * Updating the Subsetting Tests by Marker section (#2272) * update release date for 5.1.0 release (#2276) * update version for development towards 6.0.0-beta1 release * added new use case that was missing * Bugfix #2279 develop - buoy station file from 2022 (#2280) * Feature 2253 series analysis tests (#2277) * Fix typo in pull_request_template.md * Make code coverage report available (#2287) * Use updated dtcenter/metplus-action-data-update tag that fixes bug that causes GHA disk to exceed capacity when too many data volumes are created. Use specific commit of coveralls python action to ensure it will not change * removed climatology keyword from use cases that aren't relevant that were accidentally added with PR #1984 * update readme with current information * Feature #2282 coord_release_docs (#2288) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * continue workflow if coveralls step fails -- this typically happens on a pull request from a fork because the authentication fails * Feature 2253 system util tests (#2297) * Feature #2294 LSR use case (#2295) * Feature 2253 run util tests (#2307) * Release Guide - remove beta/rc tags/releases (#2309) * Add 'component: repository maintenance' label. Already ran the script to push this change and the NOAA OAR reporting label to all the METplus repos. * Hotfix for labels, adding a new one for 'component: input data' and also fixing the get_lablels.sh to search for up to 200 existing labels. Also work on the log messages. * Fix typo in comment * Feature #2283 time looping consolidation (#2311) Co-authored-by: John Halley Gotway * New issue template: Update Truth (#2332) Co-authored-by: John Halley Gotway * feature #2253 tests for run_util, 'Usage' bugfix (#2313) * Feature #2338 Debian 12 Conda environments (#2341) Co-authored-by: John Halley Gotway * Feature 1488 usecase satwinds (#2324) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Feature #2283 include times (#2345) Co-authored-by: John Halley Gotway * feature #2253 tests for config_validate, mock logger in conftest (#2320) * Feature #2299 / #2310 Remove deprecated MET config env vars and TCMPRPlotter (#2347) Co-authored-by: John Halley Gotway * remove MODEL and OBTYPE to use defaults from MET config * Feature #2348 v6.0.0-beta1 release (#2351) * update version for development towards 6.0.0-beta2 release * copy libGL and libEGL files into docker image to prevent errors with geovista python package * run use that uses geovista to test * Feature #2156 release_acceptance_testing (#2352) * feature #2253 print error logs from conftest (#2358) * feature #2253 met_db_load tests (#2355) * Rename 5.1.0.lua_wcoss2 to 6.0.0.lua_wcoss2 * Update and rename 5.1.0_acorn to 6.0.0_acorn * Update 6.0.0_acorn * feature #2253 add tests for gfdl_tracker (#2354) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Create 6.0.0_gaea * Update and rename 5.1.0_hera to 6.0.0_hera * Feature #2156 release_acceptance_testing take2 (#2361) Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> * Feature #2329 Docker info in Installation Chapter (#2366) * feature_2253_extract_tiles_tests (#2368) * Feature 2253 tc pairs tests (#2369) * Feature 2253 tc csv writer (#2373) * update requirements for building docs to get around build failure where python packages required to build RTD have disappeared from the automatically generated commands from RTD * fix ReadTheDocs requirements to include pillow which is a dependency of sphinx-gallery: see https://blog.readthedocs.com/defaulting-latest-build-tools/ for more info on why this was necessary * Feature #2340 TCDiag one_time_per_file_flag (#2374) * Update and rename 5.1.0_jet to 6.0.0_jet * Added libssh * prune docker files before running use cases to prevent running out of disk space when running use cases, ci-run-all-diff * Feature 2253 command builder tests (#2378) * Feature 2253 series analysis test (#2380) * Feature 2253 py embed test (#2379) * ignore cyclone plotter from code coverage report because it will be replaced with METplotpy logic * Feature 898 headers (#2389) * changing _ to - for header consistency * changing _ to - for header consistency * updating all headers for consistency * updating all headers for consistency and adding spacing The spacing doesn't seem to be required but it is the convention that we follow for headers. * updating all headers for consistency * updating headers for consistency and adding capital letters to headers * Using the overline ### to keep index consistent with other indexes * updating all headers for consistency * update requirements for building docs to get around build failure where python packages required to build RTD have disappeared from the automatically generated commands from RTD * updating all headers and some spacing for consistency * updating headers for consistency * changing to ### for consistency * Per #898, fixed the headers for the Release Guide part, chapters, and sections. * Duplicating changes in develop branch for requirements.txt file * updating headers * Per #2669, updated header formatting to resolve errors * Per #2669, udpating header * Per #2669, udpating headers * Per #2669, udpating header * Per #2669, updated header formatting * Per #2669, update header formatting * updating headers * Per #898, removed space in front of title * Capitalizing Please * changing to just italics to match standard formatting * indenting for consistent formatting * fixing italics again * changing from note to warning for consistency * updating headers, adding some capitalizing to the headers * fixing typo Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * This file was committed but never updated or used Per Minna, ok to delete. * Restructuring table of contents to make it more clear which guides are for users and which are for contributors * fixing formatting for clairity Co-authored-by: Julie Prestopnik --------- Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Julie Prestopnik * Feature #2349 upgrade instructions for deprecated MET config files (#2386) * Feature 2123 di doc update (#2392) Co-authored-by: Tracy * change log to list name of config variable , e.g. GRID_STAT_CONFIG_FILE, to easily see which variable should be removed * Major refactor including elimination of unnecessary imports, only computing the index for the season requested instead of all seasons all the time, which also fixes a bug selecting which season the user requested. Results are identical for all seasons for the test data for the use case. * Feature 1667 doc conventions (#2397) * New additions based on the old branch feature_1667_updating_overview * Moving the release-notes reference label * Added label for METplus_glossary for use in Documentation conentions section. * Adding images for the Documentation Conventions section * Modifying wording and testing formatting in Internal Links section * Second take on formatting * Third attempt at formatting * Fourth attempt at formatting * Modified wording, sections, and formatting * Minor modifications * Added period * Changed Pretty Table to PrettyTable * Modify informationg about converting an existing table and adding images * Resolving errors * Reformatting * Moving placement of reference label * Attempting to fix table title * Fixed incorrect alignment * Made changes based on Lisa's suggestions * Made changes based on Lisa's suggestions * Made corrections * Made corrections * Per #1667, fixing typos * Per #1667, corrected text --------- Co-authored-by: Julie Prestopnik * Feature #2377 Log to terminal only (#2398) * Update conda envs to use conda-forge only (#2402) * rearrange MET config env var tables for GridStat so they are in the order that they appear in the wrapped MET config file * use mamba instead of conda and update version of netcdf python package used to 1.6.2 * skip s2s_mid_lat WeatherRegime use case that produces different results often * updated version of xesmf because <0.7.1 does not work with mamba * per #2412, fix bug in GenVxMask to put quotes around argument so a grid specification string with spaces will be an accepted value * downgrade version of esmf package to fix bug described in https://github.com/pangeo-data/xESMF/issues/246 * Feature #2219 SeriesAnalysis multiple input files (#2408) * Adding 3 new requestors to the list of common_labels.txt for NOAA/NHC, CSU/CIRA, and Australian BOM ci-skip-all * Feature 2405 match tables2wrapper (#2416) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * per #2423, follow symbolic links when traversing directories to find files within a time window, ci-run-all-diff * Feature #2252 WaveletStat wrapper (#2427) Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> * add WaveletStat use case to group and temporarily disable TCDiag use case because changes to the wrapper are needed to fix it after changes to the MET tool were merged * update version number for beta2 release (#2431) * update version for dev towards beta3 * Feature #2371 Embed use case upgrade demo video into upgrade instructions (#2444) * fix failing use case tests -- install python packages dateutil and requests via apk instead of pip because the pip commands began failing -- see PEP668 https://peps.python.org/pep-0668/ -- also changed scripts to create conda environments for use case tests to install all packages in a single call to mamba instead of individual calls * remove commands to install geovista from source because it is now installed from conda * Feature #1989: Add OMI use case (#2457) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Feature #2432 TCDiag new config changes (#2453) * move medium range 10 use case into group with 3-5 to speed up runs * Feature #2334 land_mask (and topo_mask) in PointStat (#2456) * added use cases with pygrib * Feature #2430 TCPairs consensus.diag_required and consensus.diag_min_req (#2439) * Quickfix cloud use case desc imgs (#2464) * added pics, updated desc * add last two imgs * Fixing spelling and capitalization * Feature 2454 doc overview conv (#2471) * adding documentation in different sections * adding grid table section * fixing links * grammar updates * Per #2454, updated sections and wording. * Per #2454, added a period to the end of a sentence. * Per #2454, fixing formatting * Per #2454, updating wording * adding a section for line breaks in a table * adding :code: information * trying to fix warning * take 2 * take 3 or 4 * maybe fixed * updating link * fixing web link again * web link saga continues * Changed "ReadTheDocs" to "Read the Docs" * Updated "main_v" references to be "main_v12.0" * Removed references to main_v*, replacing with raw RST It is not maintainable to have links to branches which will become old. Since we can avoid it by adding the raw RST in the documentation, I have removed all references to main_v* in favor of placing the raw RST in the documentation. * Modified the "Code in a Paragraph" section * Reworded for consistency within the document * Added back the link for Sphinx code blocks --------- Co-authored-by: Julie Prestopnik * add argument to workflow dispatch event so that MET DockerHub repo used for tests can be easily overridden to test changes in a MET PR before merging * Feature dtcenter/MET#2796 GHA Node20 deprecation warnings (#2473) * per dtcenter/MET#2796, update versions of actions to prevent deprecated node warnings in GHA runs * change arguments to workflow dispatch so they are no longer required -- these are not needed to be set when triggering by hand through the web interface * Feature dtcenter/MET#2796 develop - Fix error log artifact creation (#2475) * updated version of pillow to fix security vulnerability alerted by dependabot in PR #2477 * remove docker image after runtime image is created from metplus image and conda env image * turn on use case to test image removal * prune images if image tag doesn't exist -- it appears that if the image is built on the fly (when PR is coming from fork) then the tag exists, but if not, the image tag is set to * support commands that must run in the shell to see if || will work in docker image pruning step * try to fix image removal * Feature 2383 use case sat alt (#2480) * new docs, files for use case * new files * updating to run use case * updated python libraries, changed test env * trying new point logic * added to script for nan removal * redid Python script to take adv of new MET ability for nans * Update run status * removed unused settings * run image prune commands separately * changed shell back to false * split up use case groups so the same envs are used by a group to see if that resolves the disk space issues * turn off use cases * feature 2253 fix empty pytest logs (#2485) * added more commands to free up disk space as suggested in https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh, ci-run-all-cases * Feature 2406 redo usecase rrfs (#2488) * issue #2406 RRFS use case files * issue #2406 added usecase to tests * Issue #2406 added metplotpy and metcalcpy as dependencies * Feature #2460 allow missing input (#2493) * changed template to use datetime format that works on MacOS * update logic to only write a file list file if there are more than 1 files, updated unit tests to match new behavior, added exception handling to series analysis to prevent crash if file does not exist * use getraw instead of getstr to prevent crash if providing a filename template tag to override a config variable on the command line * Add optional argument to subset file function to always write a file list text file even if there is only 1 file found. Use this argument in UserScript wrapper so that the environment variables that contain paths to file list files are consistent in format for use in user scripts * enhanced function to support different output variable types * removed the need for overriding clear function in specific wrappers and added optional argument to skip clearing input file list * clean up formatting * per #2460, start to implement logic to prevent errors when some input files are not found * isolate logic to find input files into find_input_files functions. clean up those functions to return boolean instead of sometimes returning None or a list of files to be consistent * remove python embedding checks because MET is now smart enough to determine if a python script is used with always setting file_type * turn on use cases to test error handling * merge artifacts * run only failed cases * always run merge step * run on a case that will succeed to test error log merge step * only run error log merge step if there were 'Save error logs' jobs that succeeded * run cases that will fail * fix condition to merge error logs * run group that will succeed but have diffs - check error logs doesn't fail * testing - add use case group that will succeed but will cause diffs becaus there is no truth data - to confirm that the error log merge step behaves properly in this case * run 3 jobs, 2 should error, to confirm that error_logs is created properly * repeat diff no error test but with * per dtcenter/MET#2796, fix error log artifact creation by merging error logs if any of the 'Save error logs' steps ran successfully * run test to confirm diff does not cause merge error logs to fail * Revert "run test to confirm diff does not cause merge error logs to fail" This reverts commit ff2d1cac57c431a047ee250e9dae9b0a813a78ba. * run test to confirm error logs are merged properly when 2 use case groups have errors * try checking output variable as string instead of boolean * Revert "run test to confirm error logs are merged properly when 2 use case groups have errors" This reverts commit 8106666a73685e654e0146d4fed56f2382f1bfc7. * run test again * test again * move check for error logs for shell script and use github env vars * Revert "run test again" This reverts commit 7a0a99c6e7031c5dafb1177d4b4ca3f32a999dac. * break 2 use cases to test that error logs are still created properly * checkout repo to get script used to merge error logs * Revert "break 2 use cases to test that error logs are still created properly" This reverts commit cb6d0b46db353b4b4709183be2fe7e5ce64ff5ff. * test merge error log again on no error diff run * fix script * move merge error logic back to workflow * break 2 use cases to test that error logs are still created properly * Revert "break 2 use cases to test that error logs are still created properly" This reverts commit 82aa0e11096aace3ccc2c79cd631533fc6426900. * remove testing use case group * Revert "remove python embedding checks because MET is now smart enough to determine if a python script is used with always setting file_type" This reverts commit de3b4b03a45bb871c71e770ff9e602739d6b63d5. * clean up lines * update logic to check that python embedding is set up properly to only try to set file_type automatically if it is not already set and if the wrapper is a tool that supports multiple input files via python embedding (which require file_type to be set). also changed error if not set properly to warning and use PYTHON_NUMPY as a default * remove run_count increment before run_at_time_once - set closer to find_input_files so run count and missing input count are consistent * return boolean from find_input_files function to be consistent with other functions * per #2460, warn instead of error if missing inputs are allowed, track counters for number of runs and missing inputs * per #2460, added check to report error if allowed missing input threshold is met * run clear before running plot_data_plane * removed test group * report warning instead of error if ALLOW_MISSING_INPUTS is True * cleanup * change function to pytest fixture so it can be used by other test scripts * update ascii2nc test to process more than 1 time to ensure commands are built properly for each run * add unit tests to ensure missing input file logic works properly for ascii2nc and grid_stat * set variable to skip RuntimeFreq logic to find input files to prevent duplicate increment of run_count -- these will be removed when the wrapper has been updated to find files using RuntimeFreq logic * remove unneccesary error checking * cleanup * call function to handle input templates that need to be handled separately for each item in the comma-separated list (for UserScript and GridDiag only) * add time_info to ALL_FILES dictionaries to be consistent with other wrappers * clean up logging for reporting error when missing inputs exceeds threshold * added function to get files for a single run time to be consistent with other functions * skip increment of run_count when FIND_FILES=True and RuntimeFreq input file logic is skipped to prevent duplicate increments * added empty test files * remove redundant variables * view warnings on a failed test run * add more empty test files * added unit tests for missing input logic * remove MANDATORY setting for EnsembleStat and GenEnsProd and instead pass mandatory argument to call to find model files so warnings/errors are properly displayed for other inputs * cleanup * remove allow missing input logic from ExtractTiles wrapper * added functions to parse template/dir variables from config, removed explicit calls to read those variables from GridStat * remove error if more labels than inputs are provided (for UserScript and GridDiag only) -- extra labels will just be ignored * added required boolean for input templates * per #2460, change warning messages to debug when checking a list of DA offsets since it is common that a given offset will not always be found in the files * added tests for missing input logic for many wrappers * cleanup * fix increment of number of runs * skip missing input logic * change how required is handled for input templates * warn instead of error if missing input is allowed * remove increment of missing input counters because it is handled in RuntimeFreq * check status of input files and increment counters in overridden run_once_per_lead. remove increment of missing input counters because it is handled in run_once_per_lead * added unit tests for missing input logic * skip missing input logic * cleanup * cleanup, use fixture for tests, add unit tests for missing input, bypass missing input logic on wrappers that don't need it * removed file that is not needed * added unit tests for pb2nc to test -valid_beg/end arguments and changes to properly support any runtime frequencies * warn instead of error if allowing missing inputs * cleanup * implement changes to properly support all runtime frequencies for pb2nc. previously all files that match a wildcard will be used instead of selecting only files that fall within the specified time range. some functions moved into pb2nc wrapper will eventually be moved up so that they are used by all wrappers to be consistent * added unit tests that will fail until wrapper is updated * replace functions in RuntimeFreq wrapper used to find input files so they can be used by all wrappers, updated ioda2nc wrapper to find input files properly to fix tests * cleanup * removed mtd version of get_input_templates and added logic to RuntimeFreq's version to get the same behavior * added unit tests for MTD missing input checks * per #2491, add release notes for beta3 * Feature #2491 v6.0.0 beta3 (#2495) * update version for beta3 release * fixed typos in release notes * update version to note development towards beta4 release * Per suggestion from @JohnHalleyGotway, create intermediate branch for updating truth data to avoid branch protection rules. I added a step to delete the intermediate branch locally if it exists to prevent conflicts with the update * added quotes to prevent error in echo caused by parenthesis * fix incorrect command * Revert "fix incorrect command" This reverts commit e7dffb6b0b351ab1b4bca5b563c1f5beef7737a9. * Revert "added quotes to prevent error in echo caused by parenthesis" This reverts commit c1cb3c4f0d7851bea720a50fac6011cd381017dc. * Revert "Per suggestion from @JohnHalleyGotway, create intermediate branch for updating truth data to avoid branch protection rules. I added a step to delete the intermediate branch locally if it exists to prevent conflicts with the update" This reverts commit 525809dc3bd73ace969b046062967796035f4d86. * Hotfix: Allow symbolic link to run_metplus.py to run (#2500) * Adding use case tests * Changing test environment * Testing environment changes * Documentation update * Updating Documentation * Updating documentation for disk space failure * Added new use case category * Fixing use case test * Fixing bug in use case file * Testing s2s after data removal * add back use cases that were accidentally removed * fix incorrect use case added * Setting tests to false for merge * Removes extraneous imports. * Switches to function call for the coupling index. * Correct number of args in comment. * Testing for old use cases * Setting tests to false for merge * update tests to update develop data -- modified commands to create new use case category directory if it does not already exist, move step to remove old data to be completed just after new data is copied to vX.Y * Summation has to have a dimension supplied for the gridded data, but for pandas the only dimension is time (but it is un-named). Therefore the numerator for the covariance term had to be split out between the fcst and obs case. * Feature 2463 modify table (#2508) * creating test dropdown menus * fixing warnings * fixing warnings * fixing warnings 3 * fixing warnings 4 * Attempt to fix documentation errors * adding 2 more test dropdowns please note. There is still a message about WARNING: Duplicate explicit target name: "gridstat: cloud fractions with neighborhood and probabilities (pygrib)". John O will fix this. I should not touch it. * fixing spacing * trying to fix link * take 2 * Removing double underscores added earlier * moving dropdown menus * Adding version to dropdown menu title * fixing spacing * dropdowns date util, eofs, h5py * fixing formatting * fixing formatting * Per #2463, adding template for future entries * adding imageio, lxml & matplotlib * dropdown up to nc-time-axis * fixing spacing problems * Fixing broken s2s links and other incorrect links * Fixing spelling and capitalization * Removing the dash in front of 1.4 for nc-time-axis * Modifying formatting * adding dropdowns thru pylab * fixing problems * dropdowns thru scikit-learn * fixing spacing * final dropdowns thru yaml * fixing spacing * fixing loose ends * Per #2463, moving information to an Appendix and adding text and links in an overview * Per #2463, adding to index.rst * Per #2463, reworded language and updated Python 3.8 reference * Per #2463, fixing errors * Per #2463, made updates based on feedback at the METplus Engineering meeting. * removing tables, changing most METplus wrappers, version numbers. * trying to fix met_version * Per #2463, adding necessary code for substitutions * Per #2463, fixing syntax error * adding period * removing section 1.5 * Per #2463, replace old label reference with new label reference and updated text to reflect the move to drop down menus * read python version from file to replace in docs * fixed typo in variable name * adding python_version to the overview. --------- Co-authored-by: Julie Prestopnik Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * per #2509, automate MET version used in documentation to be X+6.Y.0 of METplus version * Bugfix #2520 ASCII2NC file window issue and redundant wrapper initialization (#2522) * per PyCharm documentation, only ignore workspace.xml idea file and commit the rest of the .idea files to version control * per #2520, create function to get METplus wrapper class without initializing it and use that function to read list of deprecated env vars to prevent redundant initialization of wrappers that can cause unintended side effects * handle file_window variables consistently by using CommandBuilder function * comment out optional config variables that previously caused a failure when unset * Update update_truth.md Fix typo in the update_truth issue template in the develop branch. * Update update_truth.md Update wording in the update_truth issue template. * Feature #2530 dev timeline (#2532) * Per #2530, add a development timeline to the METplus Release Information section of the User's Guide. Also update the Release Guide instructions. * Per #2530, tweak the wording. * Update docs/Release_Guide/release_steps/update_release_notes_development.rst Co-authored-by: Julie Prestopnik --------- Co-authored-by: Julie Prestopnik * Adds static station lookup file for use with Python embedding for FLUXNET observations. * Major overhaul to forecast Python embedding script for the TCI use case. * Major overhaul to observation Python embedding script for the TCI use case, to compute TCI from raw observations rather than read pre-computed TCI. * Updates documentation file for TCI use case. * Adds METcalcpy version number. * Refactors wording and fixes typo. * Fixes RST formatting. * Finally fixed RST error. * Adds support to remove leap days if requested. * Updates command line args for Python embedding scripts. * Feature #2537 develop sonarqube_gha (#2541) * Per #2537, add SonarQube workflow for METplus * Per #2537, update nightly build email list. * Per #2537, fix cut/paste error configure_sonarqube.sh * Per #2537, exclude test code from code coverage statistics. * Updated conf file for use case. * Removes new TCI function because it is in METcalcpy now. * Removes old code, somsome reorganization and clarification and setting of params, and also switches the fluxnet metadata file to a command line argument instead of an environment variable. * Update the 6.0.0 Coordinated Release development timeline in release-notes.rst * Support for environment variables or default options for filtering and filename patterns, DEBUG mode added and set to False by default, adjustment of print statements for logging, and refactoring filtering of stations to ensure we don't process a file that we shouldn't by better coupling of filenames and stations. * Makes DEBUG an env var for config via metplus wrappers. * Reorganization of config file, adds environment variables, and updates comments for use case changes. * Updates to documentation. * Fixes tables. * Adds table of contents to the top for users to click on. * Updates use case documentation file. * Updated config file with obs and fcst subirectories in the path. * Added optional key/value to use_case_groups.json to prevent a use case group from running to easily disable it temporarily. Disable short_range:14 use case until it can be fixed with #2551 * update pillow version based on recommendation from dependabot: https://github.com/dtcenter/METplus/security/dependabot/5 * Switches to using metplotpy_env to get metcalcpy dependency. * Adds filtering based on missing data values. * Finishing touches to debug statements for testing. * Fixing a few minor code smells from last week. * update link to METplus Components Python Requirements table in PR template * Update docs/use_cases/model_applications/land_surface/PointStat_fcstCESM_obsFLUXNET2015_TCI.py Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Feature #2555 v6.0.0 beta4 (#2556) * update version for beta4 release * added release notes for beta4 release * update version for development towards beta5 release * update location of METviewer docker-compose.yml file that moved from PR dtcenter/METviewer#525 * Feature #2537 develop single_sq_project (#2558) * Update the beta4 release date wording * Feature #2433 Ugrid config variables in GridStat/PointStat (#2517) * update version for release * added new use case that was missing * Bugfix #2279 main_v5.1 - buoy station file from 2022 (#2281) * Fix typo in pull_request_template.md * added notes to make it clear that upgrade instructions are not needed if upgrading from 5.0 to 5.1 * New issue template: Update Truth (#2332) Co-authored-by: John Halley Gotway (cherry picked from commit 44335f33ab152a0b254041961a41895dde614ae0) * add GitHub Actions workflow that is used to create Docker images that hold the conda environment used for the automated tests -- adding this to the default main_v5.1 branch so that it will become available to run for other branches as a workflow_dispatch workflow * Per #2433, added support for setting the ugrid MET config variables for GridStat and PointStat wrappers. Also moved the seeps variable up so that it matches the order of the default config files in the MET repo * add argument to workflow dispatch event so that MET DockerHub repo used for tests can be easily overridden to test changes in a MET PR before merging * Feature dtcenter/MET#2796 main_v5.1 GHA Node20 deprecation warnings (#2474) * per dtcenter/MET#2796, update versions of actions to prevent deprecated node warnings in GHA runs - main_v5.1 * fix ReadTheDocs requirements to include pillow which is a dependency of sphinx-gallery: see https://blog.readthedocs.com/defaulting-latest-build-tools/ for more info on why this was necessary * install python packages via apk instead of pip to prevent GHA failures that were fixed in develop but not in main_v5.1 * per dtcenter/MET#2796, fix error log merging for main_v5.1 same as develop * Bump pillow from 10.0.1 to 10.2.0 in /docs (#2477) Bumps [pillow](https://github.com/python-pillow/Pillow) from 10.0.1 to 10.2.0. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/10.0.1...10.2.0) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Create 5.1.0_casper * per #2433, add support for setting the optional -config argument for a ugrid config file for PointStat and GridStat. Also moved the optional arguments to be added to the command after all of the required arguments so the command is easier to read * per #2433 and discussion on meeting 3/21/2024, change command line argument from -config to -ugrid_config * update unit tests to check for new command line argument name -ugrid_config * Updates information about GDAS surface winds having a QC value that is above the default settings in the PB2NC config file. --------- Signed-off-by: dependabot[bot] Co-authored-by: John Halley Gotway Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Daniel Adriaansen * SonarQube add python code coverage report (#2565) * run code coverage before SonarQube scan * generate xml report and configure SQ to read coverage.xml * exclude more files from code coverage report * exclude more files that should not be included in the code coverage report * more changes to code coverage exclude list * removed bad characters accidentally added * exclude cyclone plotter wrapper because it is excluded from code coverage report * ignore SonarQube lint files generated by PyCharm * Updating MTD conv radius/thresh description (#2566) * Updating MTD conv radius/thresh description * Update glossary.rst * Update docs/Users_Guide/glossary.rst Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Update docs/Users_Guide/glossary.rst Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> --------- Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Feature update modulefiles (#2564) * Adding 6.0.0 file for derecho and removing 5.1.0 file for cheyenne * Updating modulefiles * Updating orion file * Adding file for hercules * Adding file for casper * Update internal/scripts/installation/modulefiles/6.0.0_casper Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> --------- Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * Feature #1989: Add OMI to Verification Datasets Guide (#2585) Refs: #1989 * Feature #2479 time_offset_warning (#2587) * Per #2479, add support for setting time_offset_warning in all wrapped MET config files * Per #2479, add documentation and examples to basic use case param files for time_offset_warning * Feature 2346 develop update templates (#2589) * Per #2346, modifying issue templates * Per #2346, modifying pull request template * Per #2346, modifying entries based on suggestions by @georgemccabe * Bugfix #2567 PointStat multiple point observation files (#2582) * per #2567, properly pass multiple point observation files to point_stat using the -point_obs argument * per #2567, fix bug that caused file window logic to fail * Revert "per #2567, fix bug that caused file window logic to fail" This reverts commit 27fe8226c58e9a028a7979664bbf224296fdd6ea. * per #2567, fix bug that caused file window logic to fail * Feature #1514 MADIS2NC wrapper (#2591) * Per #1514, implement MADIS2NC wrapper and added example use case. Also updated the function to handle the time_summary dictionary in MET config files to support names that exactly match the name found in the dictionary, e.g. ASCII2NC_TIME_SUMMARY_OBS_VAR sets time_summary.obs_var (previously only ASCII2NC_TIME_SUMMARY_VAR_NAMES was supported and is still supported) * remove execute permissions from image files * Per #1514, add image for basic use case * removed large image files that are no longer being used in documentation * add support for time_offset_warning for MADIS2NC wrapper after the PR to add that support for other wrappers has been merged into develop * report error if output template is not defined * update contributor's guide with more up-to-date info on how to create a new wrapper and basic components of wrappers * fix warnings in documentation * fix formatting issues * Per #1514, add new basic use case to automated test suite * add step to comment out version number in wrapped MET config file * turn off use case to prepare for PR * added a pytest fixture to handle comparison of use case commands and environment variable values to remove a lot of redundant logic in each wrapper test. Added fake madis data * removed commented code * properly substitute template tags in all command line arguments * properly handle unset rec_beg and rec_end to prevent missing value from being added to command lien arg * added new madis2nc use case to existing met_tool_wrapper and temporarily disabled land_surface:0 until we can resolve the differences * Feature 2346 develop update templates (#2594) * Per #2346, modifying issue templates * Per #2346, modifying pull request template * Per #2346, modifying entries based on suggestions by @georgemccabe * Per 2346, making updates based on feedback at last MET Eng. Meeting * Enhance update truth data workflow to create a uniquely named branch to update *-ref branches and commit/append to a log file that tracks the reasons for updating the truth data. This is done to ensure that the *-ref branch testing workflow run that actually updates the truth data is always run even if there are no other changes to the METplus branch since the last update, e.g. when a change to another component like MET warrants the truth data update * git add change log file in case it doesn't already exist * changed logic to no longer push changes to develop/main_vX.Y, but instead merge changes from -ref into the update branch * retain update truth history file from *-ref * dtcenter/MET#2899 fixes a bug reading point observations using Python Embedding in PointStat and EnsembleStat, which should fix the PBL use case -- dtcenter/METplus#2246 -- so turned on diff test for PBL use case to ensure that results are consistent going forward * Feature #2429 MvMODE multivar intensity (#2603) * Feature #2547 ASCII2NC -valid_beg and -valid_end arguments (#2606) * prevent divide by zero if run_count is 0 * Per #2547, add support for setting -valid_beg and -valid_end command line arguments. Added changes to make ASCII2NC wrapper able to run for all runtime frequencies * Refactored system_util preprocess_file function to reduce cognitive complexity and add quotation marks around 'filenames' that contain spaces (typically python embedding commands) so explicit handling of these cases don't need to be done downstream. Added unit tests to test more python embedding cases * remove logic to add quotes around input file since it is handled already in preprocess_file * changed find_input_files function to return None or a time_info dictionary to be consistent across wrappers * added ReformatPointWrapper to use as parent class for ASCII2NC, MADIS2NC, PB2NC, and Point2Grid wrappers to consistently handle tools that reformat point observation data. Moved verbosity to the end of commands * clean up pb2nc wrapper to be more consistent with other ReformatPoint wrappers * per #2547, added glossary entries for new config variables to set -valid_beg/end and added commented example to basic use case config file * added glossary entries for *_RUNTIME_FREQ variables * Per #2513, remove TC_RMW_MAX_RANGE_KM * Feature 2494 update fv3 data (#2610) * Updated for new data * Updated to match new data * Updates due to new data/updates to data * Updates due to new data/data variables * Changes due to changes to data variables, date * Updates due to changes to data * Update the date to reflect the new data (with updates to variables) * fixed error with formatting * Remove redundant instructions. * For testing * Update use_case_groups.json * Revert to original location of input data to use the same data for all three FV3 Physics Tendency use cases. * Revert to original location of data from tarball * Remove typo in file directory name * Update use_case_groups.json Returned 10-12 to follow 11 * Update use_case_groups.json revert to false for testing the short range use cases for FV3 physics tendency * Update use_case_groups.json fix alignment of opening curly brace * Update use_case_groups.json revert to original formatting * removed config variable that should be set by the user because it is specific to the user's environment --------- Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> * added entry to update truth change log: develop #2610 --------- Signed-off-by: dependabot[bot] Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: John Sharples <41682323+John-Sharples@users.noreply.github.com> Co-authored-by: root Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Dan Adriaansen Co-authored-by: jprestop Co-authored-by: Hank Fisher Co-authored-by: John Halley Gotway Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: reza-armuei <144857501+reza-armuei@users.noreply.github.com> Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: Tracy Co-authored-by: Mallory Row Co-authored-by: j-opatz Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Christina Kalb Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/update_truth_change_log.txt | 1 + .github/workflows/update_truth.yml | 22 +- docs/Users_Guide/glossary.rst | 122 +++++++--- docs/Users_Guide/wrappers.rst | 36 +-- ...stFV3_fcstOnly_PhysicsTendency_Planview.py | 10 +- ...ly_PhysicsTendency_VerticalCrossSection.py | 2 +- ...cstOnly_PhysicsTendency_VerticalProfile.py | 2 +- .../util/system_util/test_system_util.py | 4 + .../madis2nc/test_madis2nc_wrapper.py | 8 +- .../wrappers/mode/test_mode_wrapper.py | 7 +- .../wrappers/pb2nc/test_pb2nc_wrapper.py | 61 ++--- .../wrappers/point2grid/test_point2grid.py | 6 +- .../wrappers/tcrmw/test_tcrmw_wrapper.py | 3 - metplus/util/diff_util.py | 2 - metplus/util/system_util.py | 209 ++++++++++++------ metplus/wrappers/__init__.py | 1 + metplus/wrappers/ascii2nc_wrapper.py | 107 ++------- metplus/wrappers/compare_gridded_wrapper.py | 57 ++--- metplus/wrappers/ensemble_stat_wrapper.py | 10 +- metplus/wrappers/gen_ens_prod_wrapper.py | 4 +- metplus/wrappers/gen_vx_mask_wrapper.py | 6 +- metplus/wrappers/madis2nc_wrapper.py | 40 +--- metplus/wrappers/mode_wrapper.py | 13 +- metplus/wrappers/pb2nc_wrapper.py | 118 ++-------- metplus/wrappers/plot_data_plane_wrapper.py | 6 +- metplus/wrappers/plot_point_obs_wrapper.py | 8 +- metplus/wrappers/point2grid_wrapper.py | 82 ++----- metplus/wrappers/point_stat_wrapper.py | 12 +- metplus/wrappers/reformat_point_wrapper.py | 90 ++++++++ metplus/wrappers/regrid_data_plane_wrapper.py | 4 +- metplus/wrappers/runtime_freq_wrapper.py | 15 +- metplus/wrappers/tc_diag_wrapper.py | 9 +- metplus/wrappers/tc_gen_wrapper.py | 8 +- metplus/wrappers/tcrmw_wrapper.py | 12 +- parm/met_config/MODEConfig_wrapped | 7 +- parm/met_config/TCRMWConfig_wrapped | 2 - .../met_tool_wrapper/ASCII2NC/ASCII2NC.conf | 3 + .../use_cases/met_tool_wrapper/MODE/MODE.conf | 3 +- .../met_tool_wrapper/TCRMW/TCRMW.conf | 1 - ...MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf | 3 +- ...FV3_fcstOnly_PhysicsTendency_Planview.conf | 2 +- .../physics_tendency_planview.yaml | 71 +++--- ..._PhysicsTendency_VerticalCrossSection.conf | 2 +- ...ysics_tendency_vertical_cross_section.yaml | 71 +++--- ...tOnly_PhysicsTendency_VerticalProfile.conf | 2 +- .../physics_tendency_vertical_profile.yaml | 71 +++--- 46 files changed, 628 insertions(+), 707 deletions(-) create mode 100755 metplus/wrappers/reformat_point_wrapper.py diff --git a/.github/update_truth_change_log.txt b/.github/update_truth_change_log.txt index 89d2d1c5bd..f070692b02 100644 --- a/.github/update_truth_change_log.txt +++ b/.github/update_truth_change_log.txt @@ -1 +1,2 @@ [20240523_17:36:49 develop] No PR - Testing that the truth data will update even if there are no changes to the METplus repository since the last update +[20240612_13:45:41 develop] #2610 - #2610 changed the input data for 3 short range use cases (10-12). diff --git a/.github/workflows/update_truth.yml b/.github/workflows/update_truth.yml index ea937857ba..6b190e42fa 100644 --- a/.github/workflows/update_truth.yml +++ b/.github/workflows/update_truth.yml @@ -12,7 +12,7 @@ on: jobs: update_truth: - name: "Update or create truth reference branch" + name: "Update truth reference branch" runs-on: ubuntu-latest steps: - name: Check if branch is develop or main_vX.Y @@ -48,14 +48,6 @@ jobs: echo git checkout ${branch_name} git checkout ${branch_name} - # merge -ref branch into branch (favoring branch changes) - echo git merge -s ours origin/${branch_name}-ref - git merge -s ours origin/${branch_name}-ref - - # push changes to branch (develop or main_vX.Y) - echo git push origin ${branch_name} - git push origin ${branch_name} - # create unique branch name to update *-ref branch update_branch=update_${branch_name}_$(uuidgen | cut -d "-" -f1) echo "update_branch=${update_branch}" >> $GITHUB_OUTPUT @@ -64,9 +56,19 @@ jobs: echo git checkout -b ${update_branch} git checkout -b ${update_branch} + # merge -ref branch into update branch (favoring branch changes) + echo git merge -s ours origin/${branch_name}-ref + git merge -s ours origin/${branch_name}-ref + + change_log_path=.github/update_truth_change_log.txt + + # get truth change log from *-ref branch + cmd="git checkout origin/${branch_name}-ref -- ${change_log_path}" + echo $cmd + $cmd + # create or append to file to track truth data changes # and ensure that PR merge into *-ref branch triggered testing workflow - change_log_path=.github/update_truth_change_log.txt change_entry="[$(date +%Y%m%d_%H:%M:%S) ${branch_name}] ${{ github.event.inputs.pull_requests }} - ${{ github.event.inputs.change_summary }}" echo "${change_entry}" >> ${change_log_path} diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index d268fdb793..29890ccfcf 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -4408,11 +4408,6 @@ METplus Configuration Glossary | *Used by:* TCRMW - TC_RMW_MAX_RANGE_KM - Specify the value for 'max_range_km' in the MET configuration file for TCRMW. - - | *Used by:* TCRMW - TC_RMW_DELTA_RANGE_KM Specify the value for 'delta_range_km' in the MET configuration file for TCRMW. @@ -5188,11 +5183,6 @@ METplus Configuration Glossary | *Used by:* TCStat - USER_SCRIPT_RUNTIME_FREQ - Frequency to run the user-defined script. See :ref:`Runtime_Freq` for more information. - - | *Used by:* UserScript - USER_SCRIPT_COMMAND User-defined command to run. Filename template tags can be used to modify the command for each execution. See :term:`USER_SCRIPT_RUNTIME_FREQ` for @@ -5208,17 +5198,6 @@ METplus Configuration Glossary USER_SCRIPT_SKIP_TIMES .. warning:: **DEPRECATED:** Please use :term:`USER_SCRIPT_SKIP_VALID_TIMES`. - GRID_DIAG_RUNTIME_FREQ - Frequency to run Grid-Diag. See :ref:`Runtime_Freq` for more information. - - | *Used by:* GridDiag - - SERIES_ANALYSIS_RUNTIME_FREQ - Frequency to run SeriesAnalysis. See :ref:`Runtime_Freq` for more information. - - | *Used by:* SeriesAnalysis - - SERIES_ANALYSIS_RUN_ONCE_PER_STORM_ID If True, run SeriesAnalysis once for each storm ID found in the .tcst (TCStat output) file specified with :term:`SERIES_ANALYSIS_TC_STAT_INPUT_DIR` and :term:`SERIES_ANALYSIS_TC_STAT_INPUT_TEMPLATE`. @@ -6118,11 +6097,6 @@ METplus Configuration Glossary | *Used by:* SeriesAnalysis - MET_DB_LOAD_RUNTIME_FREQ - Frequency to run Grid-Diag. See :ref:`Runtime_Freq` for more information. - - | *Used by:* GridDiag - MET_DATA_DB_DIR Set this the location of the dtcenter/METdataio repository. @@ -9692,11 +9666,6 @@ METplus Configuration Glossary | *Used by:* PlotPointObs - PLOT_POINT_OBS_RUNTIME_FREQ - Frequency to run PlotPointObs. See :ref:`Runtime_Freq` for more information. - - | *Used by:* PlotPointObs - PLOT_POINT_OBS_MET_CONFIG_OVERRIDES Override any variables in the MET configuration file that are not supported by the wrapper. This should be set to the full variable name @@ -10420,7 +10389,15 @@ METplus Configuration Glossary | *Used by:* TCDiag MODE_MULTIVAR_INTENSITY_FLAG - Specify the value for 'multivar_intensity_flag' in the MET configuration file for MODE. + .. warning:: **DEPRECATED:** Please use :term:`MODE_MULTIVAR_INTENSITY_COMPARE_FCST` and :term:`MODE_MULTIVAR_INTENSITY_COMPARE_OBS` instead. + + MODE_MULTIVAR_INTENSITY_COMPARE_FCST + Specify the value for 'multivar_intensity_compare_fcst' in the MET configuration file for MODE. + + | *Used by:* MODE + + MODE_MULTIVAR_INTENSITY_COMPARE_OBS + Specify the value for 'multivar_intensity_compare_obs' in the MET configuration file for MODE. | *Used by:* MODE @@ -11921,3 +11898,84 @@ METplus Configuration Glossary Specify the value for 'time_offset_warning' in the MET configuration file for WaveletStat. | *Used by:* WaveletStat + + ASCII2NC_VALID_BEG + Specify the value for the command line argument '-valid_beg' for ASCII2NC. + + | *Used by:* ASCII2NC + + ASCII2NC_VALID_END + Specify the value for the command line argument '-valid_end' for ASCII2NC. + + | *Used by:* ASCII2NC + + ASCII2NC_RUNTIME_FREQ + Frequency to run ASCII2NC. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_FOR_EACH. All runtime frequencies are supported. + + | *Used by:* ASCII2NC + + GRID_DIAG_RUNTIME_FREQ + Frequency to run Grid-Diag. See :ref:`Runtime_Freq` for more information. + + | *Used by:* GridDiag + + IODA2NC_RUNTIME_FREQ + Frequency to run IODA2NC. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_FOR_EACH. All runtime frequencies are supported. + + | *Used by:* IODA2NC + + MADIS2NC_RUNTIME_FREQ + Frequency to run MADIS2NC. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_FOR_EACH. All runtime frequencies are supported. + + | *Used by:* MADIS2NC + + MET_DB_LOAD_RUNTIME_FREQ + Frequency to run Grid-Diag. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE. All runtime frequencies are supported. + + | *Used by:* GridDiag + + MTD_RUNTIME_FREQ + Frequency to run MTD. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_PER_INIT_OR_VALID. All runtime frequencies are supported. + + | *Used by:* MTD + + PB2NC_RUNTIME_FREQ + Frequency to run PB2NC. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_FOR_EACH. All runtime frequencies are supported. + + | *Used by:* PB2NC + + PLOT_POINT_OBS_RUNTIME_FREQ + Frequency to run PlotPointObs. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_FOR_EACH. All runtime frequencies are supported. + + | *Used by:* PlotPointObs + + SERIES_ANALYSIS_RUNTIME_FREQ + Frequency to run SeriesAnalysis. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE_PER_INIT_OR_VALID. All runtime frequencies are supported. + + | *Used by:* SeriesAnalysis + + STAT_ANALYSIS_RUNTIME_FREQ + Frequency to run STATAnalysis. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE. All runtime frequencies are supported. + + | *Used by:* STATAnalysis + + TC_PAIRS_RUNTIME_FREQ + Frequency to run TCPairs. See :ref:`Runtime_Freq` for more information. + Defaults to RUN_ONCE. All runtime frequencies are supported. + + | *Used by:* TCPairs + + USER_SCRIPT_RUNTIME_FREQ + Frequency to run the user-defined script. See :ref:`Runtime_Freq` for more information. + There is no default, so a value must be specified. All runtime frequencies are supported. + + | *Used by:* UserScript diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 7f06973a12..cb24c991dd 100644 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -4587,7 +4587,8 @@ METplus Configuration | :term:`FCST_MODE_IS_PROB` | :term:`FCST_MODE_PROB_IN_GRIB_PDS` | :term:`MODE_MULTIVAR_LOGIC` -| :term:`MODE_MULTIVAR_INTENSITY_FLAG` +| :term:`MODE_MULTIVAR_INTENSITY_COMPARE_FCST` +| :term:`MODE_MULTIVAR_INTENSITY_COMPARE_OBS` | :term:`FCST_MODE_VAR_NAME` | :term:`FCST_MODE_VAR_LEVELS` | :term:`FCST_MODE_VAR_THRESH` @@ -4738,8 +4739,20 @@ ${METPLUS_MULTIVAR_LOGIC} * - :term:`MODE_MULTIVAR_LOGIC` - multivar_logic -${METPLUS_MULTIVAR_INTENSITY_FLAG} -"""""""""""""""""""""""""""""""""" +${METPLUS_MULTIVAR_INTENSITY_COMPARE_FCST} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :widths: 5 5 + :header-rows: 1 + + * - METplus Config(s) + - MET Config File + * - :term:`MODE_MULTIVAR_INTENSITY_COMPARE_FCST` + - multivar_intensity_compare_fcst + +${METPLUS_MULTIVAR_INTENSITY_COMPARE_OBS} +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :widths: 5 5 @@ -4747,8 +4760,8 @@ ${METPLUS_MULTIVAR_INTENSITY_FLAG} * - METplus Config(s) - MET Config File - * - :term:`MODE_MULTIVAR_INTENSITY_FLAG` - - multivar_intensity_flag + * - :term:`MODE_MULTIVAR_INTENSITY_COMPARE_OBS` + - multivar_intensity_compare_obs ${METPLUS_FCST_FIELD} """"""""""""""""""""" @@ -10356,7 +10369,6 @@ METplus Configuration | :term:`TC_RMW_REGRID_CENSOR_VAL` | :term:`TC_RMW_N_RANGE` | :term:`TC_RMW_N_AZIMUTH` -| :term:`TC_RMW_MAX_RANGE_KM` | :term:`TC_RMW_DELTA_RANGE_KM` | :term:`TC_RMW_SCALE` | :term:`TC_RMW_STORM_ID` @@ -10616,18 +10628,6 @@ ${METPLUS_N_AZIMUTH} * - :term:`TC_RMW_N_AZIMUTH` - n_azimuth -${METPLUS_MAX_RANGE_KM} -""""""""""""""""""""""" - -.. list-table:: - :widths: 5 5 - :header-rows: 1 - - * - METplus Config(s) - - MET Config File - * - :term:`TC_RMW_MAX_RANGE_KM` - - max_range_km - ${METPLUS_DELTA_RANGE_KM} """"""""""""""""""""""""" diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py index 09c3176765..f50b4bd514 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.py @@ -37,8 +37,8 @@ # * Forecast dataset: FV3 3-D history file with physics and dynamics tendencies # * Grid specification: FV3 2-D grid specification file with latitude and longitude of each grid point # -# **Location:** All of the input data required for this use case can be -# found in the met_test sample data tarball. Click here to the METplus +# **Location:** All the input data required for this use case can be +# found in the met_test sample data tarball. Click here to access the METplus # releases page and download sample data for the appropriate release: # https://github.com/dtcenter/METplus/releases # The tarball should be unpacked into the directory that you will set @@ -49,7 +49,8 @@ ############################################################################## # External Dependencies # --------------------- -# You will need to use a versio of Python 3.86 that has the following packages +# You will need to use a compatible version of Python (for this METplus +# version) that has the following packages # installed: # # * cartopy (0.20.3 only) @@ -59,7 +60,6 @@ # * pandas # * shapely # * xarray -# @@ -75,7 +75,7 @@ # ---------------- # # This use case does not loop but plots physics tendency data that has been -# subsetted to one date: 2019-05-04. +# subsetted to one date: 2019-06-15. ################################################################################################### # METplus Configuration diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py index 3a4036f197..dce157d7bf 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.py @@ -70,7 +70,7 @@ # ---------------- # # This use case does not loop but plots physics tendency data that has been -# subsetted to one date: 2019-05-04. +# subsetted to one date: 2019-06-15. # diff --git a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py index 33a4cb9f84..16629cf3b5 100644 --- a/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py +++ b/docs/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.py @@ -80,7 +80,7 @@ # # # | This use case does not loop but plots physics tendency data that has been -# | subsetted to one date: 2019-05-04. +# | subsetted to one date: 2019-06-15. # | diff --git a/internal/tests/pytests/util/system_util/test_system_util.py b/internal/tests/pytests/util/system_util/test_system_util.py index 14f78f9e1e..ed370f2e1b 100644 --- a/internal/tests/pytests/util/system_util/test_system_util.py +++ b/internal/tests/pytests/util/system_util/test_system_util.py @@ -145,6 +145,10 @@ def test_preprocess_file_stage(metplus_config, filename, ext): ('/some/path/PYTHON_NUMPY', None, False, 'PYTHON_NUMPY'), ('/some/path/PYTHON_XARRAY', None, False, 'PYTHON_XARRAY'), ('/some/path/PYTHON_PANDAS', None, False, 'PYTHON_PANDAS'), + # python embedding command + ('/some/path/pyscript.py /some/path/input_file', 'PYTHON', False, '"/some/path/pyscript.py /some/path/input_file"'), + ('/some/path/pyscript.py /some/path/input_file', None, False, '"/some/path/pyscript.py /some/path/input_file"'), + ('PYTHON_NUMPY=scripts/python/examples/read_ascii_point.py data/sample_obs/ascii/sample_ascii_obs.txt', None, False, '"PYTHON_NUMPY=scripts/python/examples/read_ascii_point.py data/sample_obs/ascii/sample_ascii_obs.txt"'), ] ) @pytest.mark.util diff --git a/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py b/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py index 3f3c2622bd..59c2346968 100644 --- a/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/madis2nc/test_madis2nc_wrapper.py @@ -173,10 +173,10 @@ def test_madis2nc_wrapper(metplus_config, config_overrides, extra_args += f" -{optional_arg} {config_overrides[f'MADIS2NC_{optional_arg.upper()}']}" expected_cmds = [ - (f"{app_path} {verbosity} {input_dir}/{input_file1} {output_dir}/{output_file1} " - f"-type {in_type} -config {config_file}{extra_args}"), - (f"{app_path} {verbosity} {input_dir}/{input_file2} {output_dir}/{output_file2} " - f"-type {in_type} -config {config_file}{extra_args}"), + (f"{app_path} {input_dir}/{input_file1} {output_dir}/{output_file1} " + f"-type {in_type} -config {config_file}{extra_args} {verbosity}"), + (f"{app_path} {input_dir}/{input_file2} {output_dir}/{output_file2} " + f"-type {in_type} -config {config_file}{extra_args} {verbosity}"), ] compare_command_and_env_vars(all_commands, expected_cmds, env_var_values, wrapper) diff --git a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py index f64953e87a..cf2806c2e4 100644 --- a/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py +++ b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py @@ -366,8 +366,11 @@ def test_mode_missing_inputs(metplus_config, get_test_data_dir, ({'MODE_MASK_MISSING_FLAG': 'BOTH', }, {'METPLUS_MASK_MISSING_FLAG': 'mask_missing_flag = BOTH;'}), - ({'MODE_MULTIVAR_INTENSITY_FLAG': 'false, true,true', }, - {'METPLUS_MULTIVAR_INTENSITY_FLAG': 'multivar_intensity_flag = [FALSE, TRUE, TRUE];'}), + ({'MODE_MULTIVAR_INTENSITY_COMPARE_FCST': '2, 3', }, + {'METPLUS_MULTIVAR_INTENSITY_COMPARE_FCST': 'multivar_intensity_compare_fcst = [2, 3];'}), + + ({'MODE_MULTIVAR_INTENSITY_COMPARE_OBS': '1,3', }, + {'METPLUS_MULTIVAR_INTENSITY_COMPARE_OBS': 'multivar_intensity_compare_obs = [1, 3];'}), ({'MODE_FCST_MULTIVAR_NAME': 'Snow', }, {'METPLUS_FCST_MULTIVAR_NAME': 'multivar_name = "Snow";'}), diff --git a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py index 1622bb6c9a..958806cb2d 100644 --- a/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py +++ b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py @@ -75,38 +75,6 @@ def test_pb2nc_missing_inputs(metplus_config, get_test_data_dir, missing, assert wrapper.errors == errors -# --------------------- -# test_get_command -# test that command is generated correctly -# --------------------- -@pytest.mark.parametrize( - # list of input files - 'infiles', [ - ['file1'], - ['file1', 'file2'], - ['file1', 'file2', 'file3'], - ] -) -@pytest.mark.wrapper -def test_get_command(metplus_config, infiles): - pb = pb2nc_wrapper(metplus_config) - pb.outfile = 'outfilename.txt' - pb.outdir = pb.config.getdir('OUTPUT_BASE') - outpath = os.path.join(pb.outdir, pb.outfile) - pb.infiles = infiles - config_file = pb.c_dict['CONFIG_FILE'] - cmd = pb.get_command() - if not infiles: - expected_cmd = None - else: - expected_cmd = pb.app_path + ' -v 2 ' + infiles[0] + ' ' + outpath + ' ' + config_file - if len(infiles) > 1: - for infile in infiles[1:]: - expected_cmd += ' -pbfile ' + infile - - assert cmd == expected_cmd - - # --------------------- # test_find_input_files # test files can be found with find_input_files with varying offset lists @@ -146,13 +114,16 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): # unset offset in time dictionary so it will be computed del input_dict['offset'] + # recompute time_info to pass to find file functions + time_info = time_util.ti_calculate(input_dict) + # set offset list pb.c_dict['OFFSETS'] = offsets - pb.c_dict['ALL_FILES'] = pb.get_all_files_for_each(input_dict) + pb.c_dict['ALL_FILES'] = pb.get_all_files_for_each(time_info) # look for input files based on offset list - result = pb.find_input_files() + result = pb.find_input_files(time_info) # check if correct offset file was found, if None expected, check against None if offset_to_find is None: @@ -288,6 +259,14 @@ def test_find_input_files(metplus_config, offsets, offset_to_find): {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 2;'}), ({'TIME_OFFSET_WARNING': 2, 'PB2NC_TIME_OFFSET_WARNING': 4}, {'METPLUS_TIME_OFFSET_WARNING': 'time_offset_warning = 4;'}), + # 1 extra file + ({'PB2NC_INPUT_TEMPLATE': ('ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr,' + 'another_file.nr')}, {}), + # 2 extra files + ({'PB2NC_INPUT_TEMPLATE': ('ndas.t{da_init?fmt=%H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}.nr,' + 'another_file.nr,yet_another_file.nr')}, + {}), + ] ) @pytest.mark.wrapper @@ -332,20 +311,26 @@ def test_pb2nc_all_fields(metplus_config, config_overrides, env_var_values, config_file = wrapper.c_dict.get('CONFIG_FILE') out_dir = wrapper.c_dict.get('OUTPUT_DIR') + extra_file_args = '' + if 'PB2NC_INPUT_TEMPLATE' in config_overrides: + extra_files = config_overrides['PB2NC_INPUT_TEMPLATE'].split(',')[1:] + for extra_file in extra_files: + extra_file_args += f' -pbfile {input_dir}/{extra_file}' + valid_args = '' if 'PB2NC_VALID_BEGIN' in config_overrides: valid_args += f' -valid_beg {valid_beg}' if 'PB2NC_VALID_END' in config_overrides: valid_args += f' -valid_end {valid_end}' - expected_cmds = [(f"{app_path} {verbosity} " + expected_cmds = [(f"{app_path} " f"{input_dir}/ndas.t00z.prepbufr.tm12.20070401.nr " f"{out_dir}/2007033112.nc " - f"{config_file}{valid_args}"), - (f"{app_path} {verbosity} " + f"{config_file}{extra_file_args}{valid_args} {verbosity}"), + (f"{app_path} " f"{input_dir}/ndas.t12z.prepbufr.tm12.20070401.nr " f"{out_dir}/2007040100.nc " - f"{config_file}{valid_args}"), + f"{config_file}{extra_file_args}{valid_args} {verbosity}"), ] all_cmds = wrapper.run_all_times() diff --git a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py index bd4b0149d7..87a980f41e 100644 --- a/internal/tests/pytests/wrappers/point2grid/test_point2grid.py +++ b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py @@ -92,6 +92,7 @@ def test_point2grid_missing_inputs(metplus_config, get_test_data_dir, ({'POINT2GRID_QC_FLAGS': '0,1'}, ['-qc 0,1']), ({'POINT2GRID_ADP': '{valid?fmt=%Y%m}.nc'}, ['-adp 201706.nc']), ({'POINT2GRID_REGRID_TO_GRID': 'G212'}, []), + ({'POINT2GRID_REGRID_TO_GRID': 'lambert 614 428 12.190 -133.459 -95.0 12.19058 6367.47 25.0 N'}, []), ({'POINT2GRID_INPUT_LEVEL': '(*,*)'}, []), ] ) @@ -127,6 +128,9 @@ def test_point2grid_run(metplus_config, config_overrides, optional_args): config_overrides['POINT2GRID_REGRID_TO_GRID'], config_overrides['POINT2GRID_REGRID_TO_GRID'] ) + # add quotation marks around grid if it is include spaces + if len(config_overrides['POINT2GRID_REGRID_TO_GRID'].split()) > 1: + grids = [f'"{grid}"' for grid in grids] else: grids = ( os.path.join(grid_dir, '20170601.nc'), @@ -143,7 +147,7 @@ def test_point2grid_run(metplus_config, config_overrides, optional_args): expected_cmds = [] for idx in range(0, 3): expected_cmds.append( - f'{app_path} {input_files[idx]} "{grids[idx]}" {output_files[idx]}' + f'{app_path} {input_files[idx]} {grids[idx]} {output_files[idx]}' f' -field \'name="{input_name}"; level="{level}";\'' f' {extra_args}{verbosity}' ) diff --git a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py index 9961b78277..7a4d3ea6e9 100644 --- a/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py +++ b/internal/tests/pytests/wrappers/tcrmw/test_tcrmw_wrapper.py @@ -116,9 +116,6 @@ def set_minimum_config_settings(config): ({'TC_RMW_N_AZIMUTH': '12', }, {'METPLUS_N_AZIMUTH': 'n_azimuth = 12;'}), - ({'TC_RMW_MAX_RANGE_KM': '13', }, - {'METPLUS_MAX_RANGE_KM': 'max_range_km = 13.0;'}), - ({'TC_RMW_DELTA_RANGE_KM': '14', }, {'METPLUS_DELTA_RANGE_KM': 'delta_range_km = 14.0;'}), diff --git a/metplus/util/diff_util.py b/metplus/util/diff_util.py index 84e1c7cc0a..69963de88e 100755 --- a/metplus/util/diff_util.py +++ b/metplus/util/diff_util.py @@ -41,9 +41,7 @@ ] # keywords to search and skip diff tests if found in file path -# PBL use case can be removed after dtcenter/METplus#2246 is completed SKIP_KEYWORDS = [ - 'PointStat_fcstHRRR_obsAMDAR_PBLH_PyEmbed', 'CyclonePlotter/cyclone/20150301.png', 'plots/obs_elbow.png', 'plots/fcst_elbow.png', diff --git a/metplus/util/system_util.py b/metplus/util/system_util.py index 607aafc6bf..29f4e8bc19 100644 --- a/metplus/util/system_util.py +++ b/metplus/util/system_util.py @@ -171,20 +171,27 @@ def get_files(filedir, filename_regex): return sorted(file_paths) -def preprocess_file(filename, data_type, config, allow_dir=False): - """ Decompress gzip, bzip, or zip files or convert Gempak files to NetCDF - Args: - @param filename: Path to file without zip extensions - @param data_type: str of data_type for filename - @param config: Config object - @param allow_dir (optional): bool to allow 'filename' to be a - directory. Default is False. - Returns: - Path to staged unzipped file or original file if already unzipped +def _preprocess_passthrough(filename, data_type, allow_dir): + """!Check if filename should be returned without trying to decompress data. + Helper function for preprocess_file. + This is typically due to the filename being related to Python Embedding, + either: + A Python Embedding keyword (alone or at the end of a path, e.g. + /some/path/PYTHON_NUMPY), + The name starts with a Python Embedding type, e.g. + PYTHON_NUMPY=/some/path/script.py /some/path/file.nc, + The data type (*_INPUT_DATATYPE) contains the keyword PYTHON, or + The filename contains multiple strings separated by spaces and the first + string is a Python script ending in .py. + If the filename contains multiple strings, add quotation marks around it + before returning. + + @param filename string to process + @param data_type string defining the type of data or None + @param allow_dir boolean If True, return the directory + @returns string of filename or None if preprocess_file should check if the + file should be decompressed. """ - if not filename: - return None - if allow_dir and os.path.isdir(filename): return filename @@ -192,96 +199,154 @@ def preprocess_file(filename, data_type, config, allow_dir=False): if os.path.basename(filename) in PYTHON_EMBEDDING_TYPES: return os.path.basename(filename) + out_filename = None + # if filename starts with a python embedding type, return the full value for py_embed_type in PYTHON_EMBEDDING_TYPES: if filename.startswith(py_embed_type): - return filename + out_filename = filename # if _INPUT_DATATYPE value contains PYTHON, return the full value if data_type is not None and 'PYTHON' in data_type: - return filename + out_filename = filename - stage_dir = config.getdir('STAGING_DIR') + if filename.split()[0].endswith('.py'): + out_filename = filename - if os.path.isfile(filename): - # if filename provided ends with a valid compression extension, - # remove the extension and call function again so the - # file will be uncompressed properly. This is done so that - # the function will handle files passed to it with an - # extension the same way as files passed - # without an extension but the compressed equivalent exists - for ext in COMPRESSION_EXTENSIONS: - if filename.endswith(ext): - return preprocess_file(filename[:-len(ext)], data_type, config) - # if extension is grd (Gempak), then look in staging dir for nc file - if filename.endswith('.grd') or data_type == "GEMPAK": - if filename.endswith('.grd'): - stagefile = stage_dir + filename[:-3]+"nc" - else: - stagefile = stage_dir + filename+".nc" - if os.path.isfile(stagefile): - return stagefile - # if it does not exist, run GempakToCF and return staged nc file - # Create staging area if it does not exist - mkdir_p(os.path.dirname(stagefile)) - - # only import GempakToCF if needed - from ..wrappers import GempakToCFWrapper - - run_g2c = GempakToCFWrapper(config) - run_g2c.infiles.append(filename) - run_g2c.set_output_path(stagefile) - cmd = run_g2c.get_command() - if cmd is None: - config.logger.error("GempakToCF could not generate command") - return None - if config.logger: - config.logger.debug("Converting Gempak file into {}".format(stagefile)) - run_g2c.build() - return stagefile + if out_filename is None: + return None - return filename + # add quotation marks if string contains spaces + if len(out_filename.split()) > 1: + return f'"{filename}"' + return filename - # nc file requested and the Gempak equivalent exists - if os.path.isfile(filename[:-2]+'grd'): - return preprocess_file(filename[:-2]+'grd', data_type, config) +def _check_and_decompress(filename, stage_dir, config): + """!Check if file path contains extension that implies it is compressed. + Decompress file if necessary. + Supported compression extensions are gz, bz2, and zip. + + @param filename path to file to check + @param stage_dir staging directory to decompress files into + @param config METplusConfig object used for logging + """ # if file exists in the staging area, return that path - outpath = stage_dir + filename - if os.path.isfile(outpath): - return outpath + staged_filename = stage_dir + filename + if os.path.isfile(staged_filename): + return staged_filename # Create staging area directory only if file has compression extension if any([os.path.isfile(f'{filename}{ext}') for ext in COMPRESSION_EXTENSIONS]): - mkdir_p(os.path.dirname(outpath)) + mkdir_p(os.path.dirname(staged_filename)) # uncompress gz, bz2, or zip file if os.path.isfile(filename+".gz"): - if config.logger: - config.logger.debug("Uncompressing gz file to {}".format(outpath)) + config.logger.debug("Decompressing gz file to {}".format(staged_filename)) with gzip.open(filename+".gz", 'rb') as infile: - with open(outpath, 'wb') as outfile: + with open(staged_filename, 'wb') as outfile: outfile.write(infile.read()) infile.close() outfile.close() - return outpath + return staged_filename elif os.path.isfile(filename+".bz2"): - if config.logger: - config.logger.debug("Uncompressing bz2 file to {}".format(outpath)) + config.logger.debug("Decompressing bz2 file to {}".format(staged_filename)) with open(filename+".bz2", 'rb') as infile: - with open(outpath, 'wb') as outfile: + with open(staged_filename, 'wb') as outfile: outfile.write(bz2.decompress(infile.read())) infile.close() outfile.close() - return outpath + return staged_filename elif os.path.isfile(filename+".zip"): - if config.logger: - config.logger.debug("Uncompressing zip file to {}".format(outpath)) + config.logger.debug("Decompressing zip file to {}".format(staged_filename)) with zipfile.ZipFile(filename+".zip") as z: - with open(outpath, 'wb') as f: + with open(staged_filename, 'wb') as f: f.write(z.read(os.path.basename(filename))) - return outpath + return staged_filename + + return None + + +def _process_gempak_file(filename, stage_dir, config): + """!Run GempakToCFWrapper on GEMPAK file to convert it to NetCDF. + Assumes either filename ends with .grd extension or file type has been + specified as GEMPAK. + + @param filename path to file to process + @param stage_dir staging directory to write NetCDF output + @param config METplusConfig object + @returns path to staged NetCDF output file or None if conversion failed + """ + if filename.endswith('.grd'): + stage_file = stage_dir + filename[:-3] + "nc" + else: + stage_file = stage_dir + filename + ".nc" + if os.path.isfile(stage_file): + return stage_file + # if it does not exist, run GempakToCF and return staged nc file + # Create staging area if it does not exist + mkdir_p(os.path.dirname(stage_file)) + + # only import GempakToCF if needed + from ..wrappers import GempakToCFWrapper + + run_g2c = GempakToCFWrapper(config) + run_g2c.infiles.append(filename) + run_g2c.set_output_path(stage_file) + cmd = run_g2c.get_command() + if cmd is None: + config.logger.error("GempakToCF could not generate command") + return None + config.logger.debug("Converting Gempak file into {}".format(stage_file)) + run_g2c.build() + return stage_file + + +def preprocess_file(filename, data_type, config, allow_dir=False): + """Check file path to determine if it needs to be preprocessed, e.g. + decompress or convert GEMPAK to NetCDF. + + @param filename Path to file without zip extensions + @param data_type str of data_type for filename + @param config METplusConfig object + @param allow_dir (optional) bool to allow 'filename' to be a directory. + Default is False. + @returns Path to staged unzipped file or original file if already unzipped + """ + if not filename: + return None + + # check if preprocessing should be skipped and return string if so + pass_filename = _preprocess_passthrough(filename, data_type, allow_dir) + if pass_filename is not None: + return pass_filename + + stage_dir = config.getdir('STAGING_DIR') + + if os.path.isfile(filename): + # if filename provided ends with a valid compression extension, + # remove the extension and call function again so the + # file will be uncompressed properly. This is done so that + # the function will handle files passed to it with an + # extension the same way as files passed + # without an extension but the compressed equivalent exists + for ext in COMPRESSION_EXTENSIONS: + if filename.endswith(ext): + return preprocess_file(filename[:-len(ext)], data_type, config) + # if extension is grd (Gempak), then look in staging dir for nc file + if filename.endswith('.grd') or data_type == "GEMPAK": + return _process_gempak_file(filename, stage_dir, config) + + return filename + + # nc file requested and the Gempak equivalent exists + if os.path.isfile(filename[:-2]+'grd'): + return preprocess_file(filename[:-2]+'grd', data_type, config) + + staged_path = _check_and_decompress(filename, stage_dir, config) + if staged_path is not None: + return staged_path # if input doesn't need to exist, return filename if not config.getbool('config', 'INPUT_MUST_EXIST', True): diff --git a/metplus/wrappers/__init__.py b/metplus/wrappers/__init__.py index 657523577e..c9d53693ba 100644 --- a/metplus/wrappers/__init__.py +++ b/metplus/wrappers/__init__.py @@ -16,6 +16,7 @@ 'runtime_freq_wrapper': 'RuntimeFreqWrapper', 'loop_times_wrapper': 'LoopTimesWrapper', 'reformat_gridded_wrapper': 'ReformatGriddedWrapper', + 'reformat_point_wrapper': 'ReformatPointWrapper', 'compare_gridded_wrapper': 'CompareGriddedWrapper', 'regrid_data_plane_wrapper': 'RegridDataPlaneWrapper', } diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 80b8272b13..bbf0300ff3 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -12,9 +12,8 @@ import os -from ..util import time_util -from . import LoopTimesWrapper -from ..util import do_string_sub, skip_time, get_lead_sequence +from . import ReformatPointWrapper +from ..util import do_string_sub '''!@namespace ASCII2NCWrapper @brief Wraps the ASCII2NC tool to reformat ascii format to NetCDF @@ -22,10 +21,10 @@ ''' -class ASCII2NCWrapper(LoopTimesWrapper): +class ASCII2NCWrapper(ReformatPointWrapper): RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' - RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + RUNTIME_FREQ_SUPPORTED = 'ALL' WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_TIME_SUMMARY_DICT', @@ -63,95 +62,27 @@ def create_c_dict(self): # don't provide wrapped config file name as default value c_dict['CONFIG_FILE'] = self.get_config_file() - c_dict['ASCII_FORMAT'] = self.config.getstr('config', - 'ASCII2NC_INPUT_FORMAT', - '') - c_dict['MASK_GRID'] = self.config.getstr('config', - 'ASCII2NC_MASK_GRID', - '') - c_dict['MASK_POLY'] = self.config.getstr('config', - 'ASCII2NC_MASK_POLY', - '') - c_dict['MASK_SID'] = self.config.getstr('config', - 'ASCII2NC_MASK_SID', - '') - c_dict['OBS_INPUT_DIR'] = self.config.getdir('ASCII2NC_INPUT_DIR', - '') - c_dict['OBS_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'ASCII2NC_INPUT_TEMPLATE') - ) - if not c_dict['OBS_INPUT_TEMPLATE']: - self.log_error("ASCII2NC_INPUT_TEMPLATE required to run") - - c_dict['OUTPUT_DIR'] = self.config.getdir('ASCII2NC_OUTPUT_DIR', '') - c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'ASCII2NC_OUTPUT_TEMPLATE') - ) + c_dict['FORMAT'] = self.config.getraw('config', 'ASCII2NC_INPUT_FORMAT') + if c_dict['FORMAT'] == 'python': + c_dict['OBS_INPUT_DATATYPE'] = 'PYTHON' + c_dict['MASK_GRID'] = self.config.getraw('config', 'ASCII2NC_MASK_GRID') + c_dict['MASK_POLY'] = self.config.getraw('config', 'ASCII2NC_MASK_POLY') + c_dict['MASK_SID'] = self.config.getraw('config', 'ASCII2NC_MASK_SID') + c_dict['VALID_BEG'] = self.config.getraw('config', 'ASCII2NC_VALID_BEG') + c_dict['VALID_END'] = self.config.getraw('config', 'ASCII2NC_VALID_END') # MET config variables self.handle_time_summary_dict() # handle file window variables self.handle_file_window_variables(c_dict, data_types=['OBS']) - # skip RuntimeFreq input file logic - remove once integrated - c_dict['FIND_FILES'] = False return c_dict - def get_command(self): - cmd = self.app_path - - # add input files - for infile in self.infiles: - cmd += ' ' + infile - - # add output path - out_path = self.get_output_path() - cmd += ' ' + out_path - - # add arguments - cmd += ''.join(self.args) - - # add verbosity - cmd += ' -v ' + self.c_dict['VERBOSITY'] - return cmd - - def find_input_files(self, time_info): - # if using python embedding input, don't check if file exists, - # just substitute time info and add to input file list - if self.c_dict['ASCII_FORMAT'] == 'python': - filename = do_string_sub(self.c_dict['OBS_INPUT_TEMPLATE'], - **time_info) - self.infiles.append(filename) - return True - - # get list of files even if only one is found (return_list=True) - obs_path = self.find_obs(time_info, return_list=True) - if obs_path is None: - return False - - self.infiles.extend(obs_path) - return True - def set_command_line_arguments(self, time_info): - # add input data format if set - if self.c_dict['ASCII_FORMAT']: - self.args.append(" -format {}".format(self.c_dict['ASCII_FORMAT'])) - - # add config file - passing through do_string_sub to get custom string if set - if self.c_dict['CONFIG_FILE']: - config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) - self.args.append(f" -config {config_file}") - - # add mask grid if set - if self.c_dict['MASK_GRID']: - self.args.append(" -mask_grid {}".format(self.c_dict['MASK_GRID'])) - - # add mask poly if set - if self.c_dict['MASK_POLY']: - self.args.append(" -mask_poly {}".format(self.c_dict['MASK_POLY'])) - - # add mask SID if set - if self.c_dict['MASK_SID']: - self.args.append(" -mask_sid {}".format(self.c_dict['MASK_SID'])) + # add all arguments if set + for arg in ('FORMAT', 'CONFIG_FILE', 'VALID_BEG', 'VALID_END', + 'MASK_GRID', 'MASK_POLY', 'MASK_SID'): + if self.c_dict[arg]: + val = do_string_sub(self.c_dict[arg], **time_info) + arg_name = 'config' if arg == 'CONFIG_FILE' else arg.lower() + self.args.append(f"-{arg_name} {val}") diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index e606be4bb3..9e27757b55 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -129,7 +129,7 @@ def find_input_files(self, time_info): mandatory=True, return_list=True) if not model_path: - return False + return None # if there is more than 1 file, create file list file if len(model_path) > 1: @@ -143,16 +143,16 @@ def find_input_files(self, time_info): self.infiles.append(model_path) # get observation to from first var compare - obs_path, time_info = self.find_obs_offset(time_info, - mandatory=True, - return_list=True) + obs_path, offset_time_info = self.find_obs_offset(time_info, + mandatory=True, + return_list=True) if obs_path is None: - return False + return None # if there is more than 1 file, create file list file if len(obs_path) > 1: - list_filename = (f"{time_info['init_fmt']}_" - f"{time_info['lead_hours']}_" + list_filename = (f"{offset_time_info['init_fmt']}_" + f"{offset_time_info['lead_hours']}_" f"{self.app_name}_obs.txt") obs_path = self.write_list_file(list_filename, obs_path) else: @@ -160,7 +160,7 @@ def find_input_files(self, time_info): self.infiles.append(obs_path) - return True + return offset_time_info def run_at_time_one_field(self, time_info, var_info): """! Build MET command for a single field for a given @@ -278,42 +278,11 @@ def get_command(self): @rtype string @return Returns a MET command with arguments that you can run """ - if self.app_path is None: - self.log_error('No app path specified. You must use a subclass') - return None - - cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY']) - - if len(self.infiles) == 0: - self.log_error("No input filenames specified") - return None - - # add forecast file - fcst_file = self.infiles[0] - if fcst_file.startswith('PYTHON'): - fcst_file = f"'{fcst_file}'" - cmd += f'{fcst_file} ' - - # add observation file - obs_file = self.infiles[1] - if obs_file.startswith('PYTHON'): - obs_file = f"'{obs_file}'" - cmd += f'{obs_file} ' - - if self.param == '': - self.log_error('Must specify config file to run MET tool') - return None - - cmd += self.param + ' ' - - for arg in self.args: - cmd += arg + " " - - if self.outdir == "": - self.log_error("No output directory specified") - return None - - cmd += '-outdir {}'.format(self.outdir) + cmd = (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" + f" {self.infiles[0]} {self.infiles[1]} {self.param}") + if self.args: + cmd += f" {' '.join(self.args)}" + cmd += f" -outdir {self.outdir}" return cmd def handle_interp_dict(self, uses_field=False): diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index d5fa2542e5..6007fbe5f8 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -398,14 +398,14 @@ def find_input_files(self, time_info): fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS') if not self.find_input_files_ensemble(time_info, fill_missing=fill_missing): - return False + return None # get point observation file if requested if self.c_dict['OBS_POINT_INPUT_TEMPLATE']: point_obs_files = self.find_data(time_info, data_type='OBS_POINT', return_list=True) if point_obs_files is None: - return False + return None for point_obs_path in point_obs_files: self.args.append(f'-point_obs "{point_obs_path}"') @@ -415,7 +415,7 @@ def find_input_files(self, time_info): grid_obs_files = self.find_data(time_info, data_type='OBS_GRID', return_list=True) if grid_obs_files is None: - return False + return None for grid_obs_path in grid_obs_files: self.args.append(f'-grid_obs "{grid_obs_path}"') @@ -425,11 +425,11 @@ def find_input_files(self, time_info): ens_mean_path = self.find_data(time_info, data_type='ENS_MEAN', return_list=True) if ens_mean_path is None: - return False + return None self.args.append(f'-ens_mean {ens_mean_path[0]}') - return True + return time_info def set_environment_variables(self, time_info): self.add_env_var("MET_OBS_ERROR_TABLE", diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index c4d4ee05b2..3ed3fcd643 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -250,8 +250,8 @@ def find_input_files(self, time_info): fill_missing = not self.env_var_dict.get('METPLUS_ENS_MEMBER_IDS') if not self.find_input_files_ensemble(time_info, fill_missing=fill_missing): - return False - return True + return None + return time_info def find_field_info(self, time_info): """! parse var list for ENS fields diff --git a/metplus/wrappers/gen_vx_mask_wrapper.py b/metplus/wrappers/gen_vx_mask_wrapper.py index 68aa313746..9d971537c6 100755 --- a/metplus/wrappers/gen_vx_mask_wrapper.py +++ b/metplus/wrappers/gen_vx_mask_wrapper.py @@ -189,7 +189,7 @@ def find_input_files(self, time_info, temp_file): # return if file was not found if not input_path: - return False + return None # if temp file is set, use that as input else: @@ -198,10 +198,10 @@ def find_input_files(self, time_info, temp_file): # find mask file, using MASK_INPUT_TEMPLATE mask_file = self.find_data(time_info, data_type='MASK') if not mask_file: - return False + return None # add input and mask file to input file list self.infiles.append(f'"{remove_quotes(input_path)}"') self.infiles.append(mask_file) - return True + return time_info diff --git a/metplus/wrappers/madis2nc_wrapper.py b/metplus/wrappers/madis2nc_wrapper.py index 420a5d4f22..45ae2a1ea3 100755 --- a/metplus/wrappers/madis2nc_wrapper.py +++ b/metplus/wrappers/madis2nc_wrapper.py @@ -13,7 +13,7 @@ import os from ..util import do_string_sub, MISSING_DATA_VALUE -from . import RuntimeFreqWrapper +from . import ReformatPointWrapper '''!@namespace MADIS2NCWrapper @brief Wraps the madis2nc tool to reformat MADIS format to NetCDF @@ -21,7 +21,7 @@ ''' -class MADIS2NCWrapper(RuntimeFreqWrapper): +class MADIS2NCWrapper(ReformatPointWrapper): RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' RUNTIME_FREQ_SUPPORTED = 'ALL' @@ -44,16 +44,6 @@ def create_c_dict(self): # file I/O c_dict['ALLOW_MULTIPLE_FILES'] = True - self.get_input_templates(c_dict, { - 'OBS': {'prefix': 'MADIS2NC', 'required': True}, - }) - - c_dict['OUTPUT_DIR'] = self.config.getdir('MADIS2NC_OUTPUT_DIR', '') - c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('config', 'MADIS2NC_OUTPUT_TEMPLATE') - ) - if not c_dict['OUTPUT_TEMPLATE']: - self.log_error('MADIS2NC_OUTPUT_TEMPLATE must be set') # config file settings c_dict['CONFIG_FILE'] = self.get_config_file('Madis2NcConfig_wrapped') @@ -78,32 +68,6 @@ def create_c_dict(self): return c_dict - def get_command(self): - """!Build command to run madis2nc - - @returns str: madis2nc command - """ - return (f"{self.app_path} -v {self.c_dict['VERBOSITY']}" - f" {' '.join(self.infiles)} {self.get_output_path()}" - f" {' '.join(self.args)}") - - def find_input_files(self, time_info): - """!Get list of input files to pass to command. Sets self.infiles. - - @param time_info dictionary containing time information - @returns bool: True if files were found, False otherwise - """ - if not self.c_dict.get('ALL_FILES'): - return False - - input_files = self.c_dict['ALL_FILES'][0].get('OBS', []) - if not input_files: - return False - - self.logger.debug(f"Adding input: {' and '.join(input_files)}") - self.infiles.extend(input_files) - return True - def set_command_line_arguments(self, time_info): """!Read self.c_dict and set command line arguments in self.args. diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 36a83c3941..13190a3b33 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -65,7 +65,8 @@ class MODEWrapper(CompareGriddedWrapper): 'METPLUS_FCST_FILE_TYPE', 'METPLUS_OBS_FILE_TYPE', 'METPLUS_MULTIVAR_LOGIC', - 'METPLUS_MULTIVAR_INTENSITY_FLAG', + 'METPLUS_MULTIVAR_INTENSITY_COMPARE_FCST', + 'METPLUS_MULTIVAR_INTENSITY_COMPARE_OBS', 'METPLUS_FCST_MULTIVAR_NAME', 'METPLUS_FCST_MULTIVAR_LEVEL', 'METPLUS_OBS_MULTIVAR_NAME', @@ -441,9 +442,13 @@ def create_c_dict(self): self.handle_mask(single_value=True, get_flags=True) - self.add_met_config(name='multivar_intensity_flag', data_type='list', - extra_args={'remove_quotes': True, - 'uppercase': True}) + self.add_met_config(name='multivar_intensity_compare_fcst', + data_type='list', + extra_args={'remove_quotes': True}) + self.add_met_config(name='multivar_intensity_compare_obs', + data_type='list', + extra_args={'remove_quotes': True}) + # skip RuntimeFreq input file logic - remove once integrated c_dict['FIND_FILES'] = False return c_dict diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index f17ab2c183..43bcd1615b 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -14,12 +14,11 @@ from ..util import getlistint from ..util import do_string_sub -from ..util import add_field_info_to_time_info -from . import LoopTimesWrapper +from . import ReformatPointWrapper -class PB2NCWrapper(LoopTimesWrapper): - """! Wrapper to the MET tool pb2nc which converts prepbufr files +class PB2NCWrapper(ReformatPointWrapper): + """!Wrapper to the MET tool pb2nc which converts prepbufr files to NetCDF for MET's point_stat tool can recognize. """ RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' @@ -65,11 +64,7 @@ def create_c_dict(self): """! Create a data structure (dictionary) that contains all the values set in the configuration files - Args: - - Returns: - c_dict - a dictionary containing the settings in the - configuration files + @returns dictionary containing the settings from the configuration files """ c_dict = super().create_c_dict() @@ -81,16 +76,6 @@ def create_c_dict(self): 'PB2NC_OFFSETS', '0')) - self.get_input_templates(c_dict, { - 'OBS': {'prefix': 'PB2NC', 'required': True}, - }) - - c_dict['OUTPUT_DIR'] = self.config.getdir('PB2NC_OUTPUT_DIR', '') - c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('config', - 'PB2NC_OUTPUT_TEMPLATE') - if not c_dict['OUTPUT_TEMPLATE']: - self.log_error('Must set PB2NC_OUTPUT_TEMPLATE in config file') - c_dict['OBS_INPUT_DATATYPE'] = ( self.config.getraw('config', 'PB2NC_INPUT_DATATYPE', '') ) @@ -115,10 +100,8 @@ def create_c_dict(self): self.handle_file_window_variables(c_dict, data_types=['OBS']) - c_dict['VALID_BEGIN_TEMPLATE'] = self.config.getraw('config', - 'PB2NC_VALID_BEGIN') - c_dict['VALID_END_TEMPLATE'] = self.config.getraw('config', - 'PB2NC_VALID_END') + c_dict['VALID_BEG'] = self.config.getraw('config', 'PB2NC_VALID_BEGIN') + c_dict['VALID_END'] = self.config.getraw('config', 'PB2NC_VALID_END') c_dict['ALLOW_MULTIPLE_FILES'] = True @@ -142,85 +125,22 @@ def create_c_dict(self): return c_dict - def find_input_files(self): - """!Find prepbufr data to convert. - - @returns time info if files are found, None otherwise - """ - if not self.c_dict.get('ALL_FILES'): - return None - - input_files = self.c_dict['ALL_FILES'][0].get('OBS', []) - if not input_files: - return None - - self.logger.debug(f"Adding input: {' and '.join(input_files)}") - self.infiles.extend(input_files) - return self.c_dict['ALL_FILES'][0].get('time_info') - - def set_valid_window_variables(self, time_info): - begin_template = self.c_dict['VALID_BEGIN_TEMPLATE'] - end_template = self.c_dict['VALID_END_TEMPLATE'] - - if begin_template: - self.c_dict['VALID_WINDOW_BEGIN'] = do_string_sub(begin_template, - **time_info) - - if end_template: - self.c_dict['VALID_WINDOW_END'] = do_string_sub(end_template, - **time_info) - - def run_at_time_once(self, input_dict): - """!Find files needed to run pb2nc and run if found""" - # look for input files to process - time_info = self.find_input_files() - - # if no files were found, don't run pb2nc - if time_info is None: - return - - # look for output file path and skip running pb2nc if necessary - if not self.find_and_check_output_file(time_info): - return - - # set environment variables to be passed to MET config file - self.set_environment_variables(time_info) - - self.set_valid_window_variables(time_info) - + def set_command_line_arguments(self, time_info): # handle config file substitution - self.c_dict['CONFIG_FILE'] = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) - - # build and run command - self.build() - - def get_command(self): - """! Builds the command to run the MET application - @rtype string - @return Returns a MET command with arguments that you can run - """ - cmd = f"{self.app_path} -v {self.c_dict['VERBOSITY']}" - - for arg in self.args: - cmd += f' {arg}' - - cmd += f" {self.infiles[0]}" + config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) + self.args.append(config_file) - out_path = self.get_output_path() - cmd += f" {out_path}" - - cmd += f" {self.c_dict['CONFIG_FILE']}" - - # add additional input files with -pbfile argument + # if more than 2 input files are provided, add them with -pbfile if len(self.infiles) > 1: for infile in self.infiles[1:]: - cmd += f" -pbfile {infile}" - - if self.c_dict.get('VALID_WINDOW_BEGIN'): - cmd += f" -valid_beg {self.c_dict['VALID_WINDOW_BEGIN']}" + self.args.append(f"-pbfile {infile}") - if self.c_dict.get('VALID_WINDOW_END'): - cmd += f" -valid_end {self.c_dict['VALID_WINDOW_END']}" + # reset infiles to only include first file + self.infiles = [self.infiles[0]] - return cmd + for beg_end in ('VALID_BEG', 'VALID_END'): + template = self.c_dict[beg_end] + if not template: + continue + template = do_string_sub(template, **time_info) + self.args.append(f"-{beg_end.lower()} {template}") diff --git a/metplus/wrappers/plot_data_plane_wrapper.py b/metplus/wrappers/plot_data_plane_wrapper.py index 362488e975..6145324128 100755 --- a/metplus/wrappers/plot_data_plane_wrapper.py +++ b/metplus/wrappers/plot_data_plane_wrapper.py @@ -132,14 +132,14 @@ def find_input_files(self, time_info): # just pass value to input file list if 'PYTHON' in self.c_dict['INPUT_TEMPLATE']: self.infiles.append(self.c_dict['INPUT_TEMPLATE']) - return True + return time_info file_path = self.find_data(time_info, return_list=False) if not file_path: - return False + return None self.infiles.append(file_path) - return True + return time_info def set_command_line_arguments(self, time_info): field_name = do_string_sub(self.c_dict['FIELD_NAME'], diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 8373b1b0c0..076811804b 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -194,7 +194,7 @@ def find_input_files(self, time_info): return_list=True, mandatory=True) if input_files is None: - return False + return None self.infiles.extend(input_files) @@ -204,17 +204,17 @@ def find_input_files(self, time_info): data_type='GRID', return_list=True) if not grid_file: - return False + return None if len(grid_file) > 1: self.log_error('More than one file found from ' 'PLOT_POINT_OBS_GRID_INPUT_TEMPLATE: ' f'{grid_file.split(",")}') - return False + return None self.c_dict['GRID_INPUT_PATH'] = grid_file[0] - return True + return time_info def set_command_line_arguments(self, time_info): """! Set all arguments for plot_point_obs command. diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index 15bbbd1374..c95a9da3d4 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -14,7 +14,7 @@ from ..util import do_string_sub from ..util import remove_quotes -from . import LoopTimesWrapper +from . import ReformatPointWrapper '''!@namespace Point2GridWrapper @brief Wraps the Point2Grid tool to reformat ascii format to NetCDF @@ -22,7 +22,7 @@ ''' -class Point2GridWrapper(LoopTimesWrapper): +class Point2GridWrapper(ReformatPointWrapper): RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] @@ -40,22 +40,6 @@ def create_c_dict(self): c_dict['ALLOW_MULTIPLE_FILES'] = False - # input and output files - c_dict['OBS_INPUT_DIR'] = self.config.getdir('POINT2GRID_INPUT_DIR', - '') - - c_dict['OBS_INPUT_TEMPLATE'] = self.config.getraw('filename_templates', - 'POINT2GRID_INPUT_TEMPLATE') - - if not c_dict['OBS_INPUT_TEMPLATE']: - self.log_error("POINT2GRID_INPUT_TEMPLATE required to run") - - c_dict['OUTPUT_DIR'] = self.config.getdir('POINT2GRID_OUTPUT_DIR', - '') - - c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('filename_templates', - 'POINT2GRID_OUTPUT_TEMPLATE') - # handle window variables [POINT2GRID_]FILE_WINDOW_[BEGIN/END] c_dict['OBS_FILE_WINDOW_BEGIN'] = \ self.config.getseconds('config', 'POINT2GRID_FILE_WINDOW_BEGIN', @@ -109,61 +93,27 @@ def create_c_dict(self): c_dict['VLD_THRESH'] = self.config.getstr('config', 'POINT2GRID_VLD_THRESH', '') - # skip RuntimeFreq input file logic - remove once integrated - c_dict['FIND_FILES'] = False - return c_dict - - def get_command(self): - cmd = self.app_path - - # don't run if no input or output files were found - if not self.infiles: - self.log_error("No input files were found") - return - if not self.outfile: - self.log_error("No output file specified") - return - - # add input files - for infile in self.infiles: - if infile.startswith('PYTHON'): - infile = f"'{infile}'" - cmd += ' ' + infile - - # add grid name - grid = remove_quotes(self.c_dict['GRID']) - cmd += f' "{grid}"' - - # add output path - out_path = self.get_output_path() - cmd += ' ' + out_path - - # add arguments - cmd += ' ' + ' '.join(self.args) - - # add verbosity - cmd += ' -v ' + self.c_dict['VERBOSITY'] - return cmd + return c_dict def find_input_files(self, time_info): """!Find input file and mask file and add them to the list of input files. Args: @param time_info time dictionary for current run time - @returns List of input files found or None if either file was not found + @returns time_info if all files were found, None otherwise """ - # get input file - # calling find_obs because we set OBS_ variables in c_dict for the input data - input_path = self.find_obs(time_info) - if input_path is None: - return False - - self.infiles.append(input_path) - - self.c_dict['GRID'] = do_string_sub(self.c_dict['GRID_TEMPLATE'], - **time_info) - - return True + offset_time_info = super().find_input_files(time_info) + if not offset_time_info: + return None + + # get grid from template and add quotes if needed + grid = do_string_sub(self.c_dict['GRID_TEMPLATE'], **offset_time_info) + grid = remove_quotes(grid) + if len(grid.split()) > 1: + grid = f'"{grid}"' + self.infiles.append(grid) + + return offset_time_info def set_command_line_arguments(self, time_info): """!Set command line arguments from c_dict diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 2ff73c601d..e81c135c3f 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -318,7 +318,7 @@ def find_input_files(self, time_info): mandatory=True, return_list=True) if not model_path: - return False + return None # if there is more than 1 file, create file list file if len(model_path) > 1: @@ -332,12 +332,12 @@ def find_input_files(self, time_info): mandatory=True, return_list=True) if obs_path is None: - return False + return None # add observation files found individually to use -point_obs argument self.infiles.extend(obs_path) - return True + return time_info def get_command(self): """! Builds the command to run point_stat @@ -345,13 +345,7 @@ def get_command(self): @return Returns a point_stat command with arguments that you can run """ fcst_file, *obs_files = self.infiles - if fcst_file.startswith('PYTHON'): - fcst_file = f"'{fcst_file}'" - obs_file = obs_files[0] - if obs_file.startswith('PYTHON'): - obs_file = f"'{obs_file}'" - cmd = (f"{self.app_path} -v {self.c_dict['VERBOSITY']} " f"{fcst_file} {obs_file} {self.param}") diff --git a/metplus/wrappers/reformat_point_wrapper.py b/metplus/wrappers/reformat_point_wrapper.py new file mode 100755 index 0000000000..c40dfb098c --- /dev/null +++ b/metplus/wrappers/reformat_point_wrapper.py @@ -0,0 +1,90 @@ +""" +Program Name: reformat_point_wrapper.py +Contact(s): George McCabe +Abstract: Builds command for and runs MET tools that reformat point obs +History Log: Initial version +Usage: +Parameters: None +Input Files: +Output Files: nc files +Condition codes: 0 for success, 1 for failure +""" + +from ..util import ti_calculate +from . import RuntimeFreqWrapper + +'''!@namespace ReformatPointWrapper +@brief Wraps the MET tools that reformat point observation data +@endcode +''' + + +class ReformatPointWrapper(RuntimeFreqWrapper): + def create_c_dict(self): + c_dict = super().create_c_dict() + app_upper = self.app_name.upper() + + # populate OBS input templates using app name + self.get_input_templates(c_dict, { + 'OBS': {'prefix': app_upper, 'required': True}, + }) + + # set output templates using app name + c_dict['OUTPUT_DIR'] = self.config.getdir(f'{app_upper}_OUTPUT_DIR', '') + c_dict['OUTPUT_TEMPLATE'] = ( + self.config.getraw('config', f'{app_upper}_OUTPUT_TEMPLATE') + ) + if not c_dict['OUTPUT_TEMPLATE']: + self.log_error(f'{app_upper}_OUTPUT_TEMPLATE must be set') + + return c_dict + + def get_command(self): + """!Build command to run + + @returns str command + """ + return (f"{self.app_path} {' '.join(self.infiles)}" + f" {self.get_output_path()}" + f"{' ' + ' '.join(self.args) if self.args else ''}" + f" -v {self.c_dict['VERBOSITY']}") + + def _get_offset_time_info(self, time_info): + """!Get offset value that was used to find input data so the output + time information can include the correct offset value. Copy the time + information dictionary, then remove the offset variables if they are + set and replace them with the computed offset from the first set of + files that were found. This is primarily needed so that PB2NC wrapper + can write output files that contain the offset. + + @param time_info dictionary containing time information from run + @returns time info dictionary with input offset value included + """ + temp_time_info = time_info.copy() + for key in ('offset', 'offset_hours'): + if key in temp_time_info: + del temp_time_info[key] + val = self.c_dict['ALL_FILES'][0].get('time_info').get(key) + if val: + temp_time_info[key] = val + + return ti_calculate(temp_time_info) + + def find_input_files(self, time_info): + if not self.c_dict.get('ALL_FILES'): + return None + + input_files = [] + for files in self.c_dict['ALL_FILES']: + new_files = files.get('OBS', []) + if not new_files: + continue + input_files.extend(new_files) + + if not input_files: + return None + + self.logger.debug(f"Adding input: {' and '.join(input_files)}") + self.infiles.extend(input_files) + + return self._get_offset_time_info(time_info) diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index 5ae51fd08b..940743800e 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -347,7 +347,7 @@ def find_input_files(self, time_info, data_type): """ input_path = self.find_data(time_info, data_type=data_type) if not input_path: - return False + return None self.infiles.append(input_path) @@ -356,7 +356,7 @@ def find_input_files(self, time_info, data_type): # put quotes around verification grid in case it is a grid description self.infiles.append(f'"{grid}"') - return True + return time_info def set_command_line_arguments(self): """!Returns False if command should not be run""" diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 67f5f37638..308e4ec711 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -191,14 +191,17 @@ def run_all_times(self): # if missing inputs are allowed, check threshold to report error if self.c_dict['ALLOW_MISSING_INPUTS']: - success_rate = (1 - (self.missing_input_count / self.run_count)) * 100 - allowed_rate = self.c_dict['INPUT_THRESH'] * 100 + if not self.run_count: + success_rate = 0 + else: + success_rate = (1 - (self.missing_input_count / self.run_count)) + allowed_rate = self.c_dict['INPUT_THRESH'] if success_rate < allowed_rate: self.log_error( - f'{success_rate}% of {wrapper_instance_name} runs had all ' - f'required inputs. Must have {allowed_rate}% to prevent error. ' - f'{self.missing_input_count} out of {self.run_count} runs ' - 'had missing inputs.' + f'{success_rate * 100}% of {wrapper_instance_name} runs had ' + f'all required inputs. Must have {allowed_rate * 100}% to ' + f'prevent error. {self.missing_input_count} out of ' + f'{self.run_count} runs had missing inputs.' ) return self.all_commands diff --git a/metplus/wrappers/tc_diag_wrapper.py b/metplus/wrappers/tc_diag_wrapper.py index b70392b599..90c22f1570 100755 --- a/metplus/wrappers/tc_diag_wrapper.py +++ b/metplus/wrappers/tc_diag_wrapper.py @@ -366,12 +366,12 @@ def find_input_files(self, time_info): """!Get DECK file and list of input data files and set c_dict items. Args: @param time_info time dictionary to use for string substitution - @returns Input file list if all files were found, None if not. + @returns time_info if all files were found, None if not. """ # get deck file deck_file = self.find_data(time_info, data_type='DECK') if not deck_file: - return False + return None self.c_dict['DECK_FILE'] = deck_file # get files and values for -data arguments @@ -379,8 +379,9 @@ def find_input_files(self, time_info): for data_dict in self.c_dict['DATA_INPUTS']: if not self._find_data_inputs(data_dict, lead_seq, time_info, deck_file): - return False - return True + return None + + return time_info def _find_data_inputs(self, data_dict, lead_seq, time_info, deck_file): # check if file list file is set and use that instead of template/dir diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index aa7ab2d558..460256686c 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -329,7 +329,7 @@ def find_input_files(self, time_info): set c_dict. @param time_info time dictionary to use for string substitution - @returns True if all inputs were found, False if not. + @returns time_info if all inputs were found, None if not. """ # get track file(s) or directory track_files = self.find_data(time_info, @@ -337,7 +337,7 @@ def find_input_files(self, time_info): return_list=True, allow_dir=True) if not track_files: - return False + return None list_filename = time_info['init_fmt'] + '_tc_gen_track.txt' self.c_dict['TRACK_FILE'] = self.write_list_file(list_filename, @@ -357,7 +357,7 @@ def find_input_files(self, time_info): # if template was provided but no files were found, skip run if not file_list: - return False + return None list_filename = f"{time_info['init_fmt']}_tc_gen_{file_type}.txt" self.c_dict[f'{file_type.upper()}_FILE'] = ( @@ -381,7 +381,7 @@ def find_input_files(self, time_info): f"lead = [{', '.join(lead_list)}];" ) - return True + return time_info def set_command_line_arguments(self, time_info): diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index 86760d574b..16aafd4379 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -44,7 +44,6 @@ class TCRMWWrapper(RuntimeFreqWrapper): 'METPLUS_REGRID_DICT', 'METPLUS_N_RANGE', 'METPLUS_N_AZIMUTH', - 'METPLUS_MAX_RANGE_KM', 'METPLUS_DELTA_RANGE_KM', 'METPLUS_RMW_SCALE', ] @@ -97,7 +96,6 @@ def create_c_dict(self): self.add_met_config(name='n_range', data_type='int') self.add_met_config(name='n_azimuth', data_type='int') - self.add_met_config(name='max_range_km', data_type='float') self.add_met_config(name='delta_range_km', data_type='float') self.add_met_config(name='rmw_scale', data_type='float') self.add_met_config(name='storm_id', data_type='string') @@ -187,7 +185,7 @@ def find_input_files(self, time_info): # get deck file deck_file = self.find_data(time_info, data_type='DECK') if not deck_file: - return False + return None self.c_dict['DECK_FILE'] = deck_file @@ -205,7 +203,7 @@ def find_input_files(self, time_info): self.logger.warning(msg) else: self.log_error(msg) - return False + return None else: all_input_files = [] @@ -224,7 +222,7 @@ def find_input_files(self, time_info): all_input_files.extend(input_files) if not all_input_files: - return False + return None # create an ascii file with a list of the input files list_file = f"{os.path.basename(deck_file)}_data_files.txt" @@ -233,11 +231,11 @@ def find_input_files(self, time_info): self.infiles.append(list_file) if not self._set_data_field(time_info): - return False + return None self._set_lead_list(time_info, lead_seq) - return True + return time_info def _set_data_field(self, time_info): """!Get list of fields from config to process. Build list of field info diff --git a/parm/met_config/MODEConfig_wrapped b/parm/met_config/MODEConfig_wrapped index 54af9bcf4b..d010683d96 100644 --- a/parm/met_config/MODEConfig_wrapped +++ b/parm/met_config/MODEConfig_wrapped @@ -54,8 +54,11 @@ ${METPLUS_QUILT} //multivar_logic = ${METPLUS_MULTIVAR_LOGIC} -//multivar_intensity_flag = -${METPLUS_MULTIVAR_INTENSITY_FLAG} +//multivar_intensity_compare_fcst = +${METPLUS_MULTIVAR_INTENSITY_COMPARE_FCST} + +//multivar_intensity_compare_obs = +${METPLUS_MULTIVAR_INTENSITY_COMPARE_OBS} // // Forecast and observation fields to be verified diff --git a/parm/met_config/TCRMWConfig_wrapped b/parm/met_config/TCRMWConfig_wrapped index 0cd32cee3a..00a7ea0ad7 100644 --- a/parm/met_config/TCRMWConfig_wrapped +++ b/parm/met_config/TCRMWConfig_wrapped @@ -76,8 +76,6 @@ ${METPLUS_REGRID_DICT} ${METPLUS_N_RANGE} //n_azimuth = ${METPLUS_N_AZIMUTH} -//max_range_km = -${METPLUS_MAX_RANGE_KM} //delta_range_km = ${METPLUS_DELTA_RANGE_KM} //rmw_scale = diff --git a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf index 778ea8ae76..f1aa331915 100644 --- a/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf +++ b/parm/use_cases/met_tool_wrapper/ASCII2NC/ASCII2NC.conf @@ -57,6 +57,9 @@ ASCII2NC_SKIP_IF_OUTPUT_EXISTS = False #LOG_ASCII2NC_VERBOSITY = 1 +#ASCII2NC_VALID_BEG = +#ASCII2NC_VALID_END = + ASCII2NC_CONFIG_FILE = {PARM_BASE}/met_config/Ascii2NcConfig_wrapped ASCII2NC_INPUT_FORMAT = diff --git a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf index 55fbba5a5f..429b71f6e6 100644 --- a/parm/use_cases/met_tool_wrapper/MODE/MODE.conf +++ b/parm/use_cases/met_tool_wrapper/MODE/MODE.conf @@ -115,7 +115,8 @@ OBTYPE = WRF MODE_CONFIG_FILE = {PARM_BASE}/met_config/MODEConfig_wrapped #MODE_MULTIVAR_LOGIC = -#MODE_MULTIVAR_INTENSITY_FLAG = +#MODE_MULTIVAR_INTENSITY_COMPARE_FCST = +#MODE_MULTIVAR_INTENSITY_COMPARE_OBS = MODE_REGRID_TO_GRID = NONE #MODE_REGRID_METHOD = diff --git a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf index 252dca5dfd..a410e91ffe 100644 --- a/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf +++ b/parm/use_cases/met_tool_wrapper/TCRMW/TCRMW.conf @@ -86,7 +86,6 @@ TC_RMW_CYCLONE = 14 #TC_RMW_N_RANGE = 100 #TC_RMW_N_AZIMUTH = 180 -#TC_RMW_MAX_RANGE_KM = 1000.0 #TC_RMW_DELTA_RANGE_KM = 10.0 #TC_RMW_SCALE = 0.2 diff --git a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf index 1f3764074d..ccf6a7a33d 100644 --- a/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf +++ b/parm/use_cases/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl.conf @@ -25,7 +25,8 @@ OBTYPE = ANALYSIS # Run MODE to output super objects MODE_MULTIVAR_LOGIC = #1 && #2 && #3 -MODE_MULTIVAR_INTENSITY_FLAG = FALSE,TRUE,TRUE +MODE_MULTIVAR_INTENSITY_COMPARE_FCST = 2, 3 +MODE_MULTIVAR_INTENSITY_COMPARE_OBS = 2, 3 FCST_MODE_INPUT_DIR = {INPUT_BASE}/model_applications/short_range/MODEMultivar_fcstHRRR_obsMRMS_HRRRanl FCST_MODE_INPUT_TEMPLATE = hrrr.t{init?fmt=%H}z.wrfprsf{lead?fmt=%H}.sub.grib2,hrrr.t{init?fmt=%H}z.wrfprsf{lead?fmt=%H}.sub.grib2,hrrr.t{init?fmt=%H}z.wrfprsf{lead?fmt=%H}.sub.grib2 diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf index 3dada0c557..33ef363700 100644 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview.conf @@ -40,7 +40,7 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE FV3_HISTORY_FILE = {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/fv3_history.nc GRID_SPEC_FILE = {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/grid_spec.nc PRESSURE_LEVEL = 500 -USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/planview_plot.py {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/physics_tendency_planview.yaml {FV3_HISTORY_FILE} {GRID_SPEC_FILE} tmp pbl -p {PRESSURE_LEVEL} -t 1 -v 20190504T14 -o {OUTPUT_BASE}/plots/short_range-physics_tendency_planview.png --nofineprint +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/planview_plot.py {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/physics_tendency_planview.yaml {FV3_HISTORY_FILE} {GRID_SPEC_FILE} tmp pbl -p {PRESSURE_LEVEL} -t 1 -v 20190615T20 -o {OUTPUT_BASE}/plots/short_range-physics_tendency_planview.png --nofineprint [user_env_vars] diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/physics_tendency_planview.yaml b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/physics_tendency_planview.yaml index 182a3f2f0f..748a2d8bff 100644 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/physics_tendency_planview.yaml +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_Planview/physics_tendency_planview.yaml @@ -2,47 +2,40 @@ # Each type of tendency (moisture, temperature, wind component) has its own set of variables. tendency_varnames: spfh: - - dq3dt_deepcnv - - dq3dt_mp - - dq3dt_pbl - - dq3dt_shalcnv - - dq3dt_nophys + - dtend_qv_pbl + - dtend_qv_deepcnv + - dtend_qv_shalcnv + - dtend_qv_mp + - dtend_qv_phys + - dtend_qv_nophys tmp: - - dt3dt_congwd - - dt3dt_deepcnv - - dt3dt_lw - - dt3dt_mp - - dt3dt_orogwd - - dt3dt_pbl - - dt3dt_rdamp - - dt3dt_shalcnv - - dt3dt_sw - - dt3dt_nophys + - dtend_temp_lw + - dtend_temp_sw + - dtend_temp_pbl + - dtend_temp_deepcnv + - dtend_temp_shalcnv + - dtend_temp_mp + - dtend_temp_orogwd + - dtend_temp_cnvgwd + - dtend_temp_phys + - dtend_temp_nophys ugrd: - - du3dt_congwd - - du3dt_deepcnv - - du3dt_mp - - du3dt_orogwd - - du3dt_pbl - - du3dt_rdamp - - du3dt_shalcnv - - du3dt_nophys + - dtend_u_pbl + - dtend_u_orogwd + - dtend_u_deepcnv + - dtend_u_cnvgwd + - dtend_u_shalcnv + - dtend_u_phys + - dtend_u_nophys vgrd: - - dv3dt_congwd - - dv3dt_deepcnv - - dv3dt_mp - - dv3dt_orogwd - - dv3dt_pbl - - dv3dt_rdamp - - dv3dt_shalcnv - - dv3dt_nophys + - dtend_v_pbl + - dtend_v_orogwd + - dtend_v_deepcnv + - dtend_v_cnvgwd + - dtend_v_shalcnv + - dtend_v_phys + - dtend_v_nophys -# Name of variables in history file that contain the temperature, moisture, wind at time zero (initialization time). -time0_varname: - tmp : tmp_i - spfh: qv_i - ugrd: ugrd_i - vgrd: vgrd_i # Name of the longitude and latitude variables in the grid specification file. @@ -65,7 +58,5 @@ standard_parallel : 38.139 cmap : "Spectral_r" # resolution (dots per inch) of output -dpi : 150 - - +dpi : 100 diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf index 3bfd4ed04c..7ad8d8f998 100644 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection.conf @@ -39,7 +39,7 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE FV3_HISTORY_FILE = {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/fv3_history.nc GRID_SPEC_FILE = {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/grid_spec.nc -USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/vertical_cross_section_plot.py {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/physics_tendency_vertical_cross_section.yaml {FV3_HISTORY_FILE} {GRID_SPEC_FILE} tmp -t 2 -v 20190504T14 -s 32 -115 -e 34 -82 -o {OUTPUT_BASE}/plots/short_range-physics_tendency_vertical_cross_section.png --nofineprint +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/vertical_cross_section_plot.py {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/physics_tendency_vertical_cross_section.yaml {FV3_HISTORY_FILE} {GRID_SPEC_FILE} tmp -t 1 -v 20190615T20 -s 32 -115 -e 34 -82 -o {OUTPUT_BASE}/plots/short_range-physics_tendency_vertical_cross_section.png --nofineprint [user_env_vars] diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/physics_tendency_vertical_cross_section.yaml b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/physics_tendency_vertical_cross_section.yaml index 182a3f2f0f..748a2d8bff 100644 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/physics_tendency_vertical_cross_section.yaml +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalCrossSection/physics_tendency_vertical_cross_section.yaml @@ -2,47 +2,40 @@ # Each type of tendency (moisture, temperature, wind component) has its own set of variables. tendency_varnames: spfh: - - dq3dt_deepcnv - - dq3dt_mp - - dq3dt_pbl - - dq3dt_shalcnv - - dq3dt_nophys + - dtend_qv_pbl + - dtend_qv_deepcnv + - dtend_qv_shalcnv + - dtend_qv_mp + - dtend_qv_phys + - dtend_qv_nophys tmp: - - dt3dt_congwd - - dt3dt_deepcnv - - dt3dt_lw - - dt3dt_mp - - dt3dt_orogwd - - dt3dt_pbl - - dt3dt_rdamp - - dt3dt_shalcnv - - dt3dt_sw - - dt3dt_nophys + - dtend_temp_lw + - dtend_temp_sw + - dtend_temp_pbl + - dtend_temp_deepcnv + - dtend_temp_shalcnv + - dtend_temp_mp + - dtend_temp_orogwd + - dtend_temp_cnvgwd + - dtend_temp_phys + - dtend_temp_nophys ugrd: - - du3dt_congwd - - du3dt_deepcnv - - du3dt_mp - - du3dt_orogwd - - du3dt_pbl - - du3dt_rdamp - - du3dt_shalcnv - - du3dt_nophys + - dtend_u_pbl + - dtend_u_orogwd + - dtend_u_deepcnv + - dtend_u_cnvgwd + - dtend_u_shalcnv + - dtend_u_phys + - dtend_u_nophys vgrd: - - dv3dt_congwd - - dv3dt_deepcnv - - dv3dt_mp - - dv3dt_orogwd - - dv3dt_pbl - - dv3dt_rdamp - - dv3dt_shalcnv - - dv3dt_nophys + - dtend_v_pbl + - dtend_v_orogwd + - dtend_v_deepcnv + - dtend_v_cnvgwd + - dtend_v_shalcnv + - dtend_v_phys + - dtend_v_nophys -# Name of variables in history file that contain the temperature, moisture, wind at time zero (initialization time). -time0_varname: - tmp : tmp_i - spfh: qv_i - ugrd: ugrd_i - vgrd: vgrd_i # Name of the longitude and latitude variables in the grid specification file. @@ -65,7 +58,5 @@ standard_parallel : 38.139 cmap : "Spectral_r" # resolution (dots per inch) of output -dpi : 150 - - +dpi : 100 diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf index 12fff6efdd..42aa513542 100644 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile.conf @@ -39,7 +39,7 @@ USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE FV3_HISTORY_FILE = {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/fv3_history.nc GRID_SPEC_FILE = {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/grid_spec.nc -USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/vertical_profile_plot.py {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/physics_tendency_vertical_profile.yaml {FV3_HISTORY_FILE} {GRID_SPEC_FILE} tmp -t 2 -v 20190504T14 -o {OUTPUT_BASE}/plots/short_range-physics_tendency_vertical_profile.png -s {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/shapefiles/MID_CONUS --nofineprint +USER_SCRIPT_COMMAND = {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/vertical_profile_plot.py {PARM_BASE}/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/physics_tendency_vertical_profile.yaml {FV3_HISTORY_FILE} {GRID_SPEC_FILE} tmp -t 1 -v 20190615T20 -o {OUTPUT_BASE}/plots/short_range-physics_tendency_vertical_profile.png -s {INPUT_BASE}/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency/shapefiles/MID_CONUS --nofineprint [user_env_vars] diff --git a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/physics_tendency_vertical_profile.yaml b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/physics_tendency_vertical_profile.yaml index 182a3f2f0f..748a2d8bff 100644 --- a/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/physics_tendency_vertical_profile.yaml +++ b/parm/use_cases/model_applications/short_range/UserScript_fcstFV3_fcstOnly_PhysicsTendency_VerticalProfile/physics_tendency_vertical_profile.yaml @@ -2,47 +2,40 @@ # Each type of tendency (moisture, temperature, wind component) has its own set of variables. tendency_varnames: spfh: - - dq3dt_deepcnv - - dq3dt_mp - - dq3dt_pbl - - dq3dt_shalcnv - - dq3dt_nophys + - dtend_qv_pbl + - dtend_qv_deepcnv + - dtend_qv_shalcnv + - dtend_qv_mp + - dtend_qv_phys + - dtend_qv_nophys tmp: - - dt3dt_congwd - - dt3dt_deepcnv - - dt3dt_lw - - dt3dt_mp - - dt3dt_orogwd - - dt3dt_pbl - - dt3dt_rdamp - - dt3dt_shalcnv - - dt3dt_sw - - dt3dt_nophys + - dtend_temp_lw + - dtend_temp_sw + - dtend_temp_pbl + - dtend_temp_deepcnv + - dtend_temp_shalcnv + - dtend_temp_mp + - dtend_temp_orogwd + - dtend_temp_cnvgwd + - dtend_temp_phys + - dtend_temp_nophys ugrd: - - du3dt_congwd - - du3dt_deepcnv - - du3dt_mp - - du3dt_orogwd - - du3dt_pbl - - du3dt_rdamp - - du3dt_shalcnv - - du3dt_nophys + - dtend_u_pbl + - dtend_u_orogwd + - dtend_u_deepcnv + - dtend_u_cnvgwd + - dtend_u_shalcnv + - dtend_u_phys + - dtend_u_nophys vgrd: - - dv3dt_congwd - - dv3dt_deepcnv - - dv3dt_mp - - dv3dt_orogwd - - dv3dt_pbl - - dv3dt_rdamp - - dv3dt_shalcnv - - dv3dt_nophys + - dtend_v_pbl + - dtend_v_orogwd + - dtend_v_deepcnv + - dtend_v_cnvgwd + - dtend_v_shalcnv + - dtend_v_phys + - dtend_v_nophys -# Name of variables in history file that contain the temperature, moisture, wind at time zero (initialization time). -time0_varname: - tmp : tmp_i - spfh: qv_i - ugrd: ugrd_i - vgrd: vgrd_i # Name of the longitude and latitude variables in the grid specification file. @@ -65,7 +58,5 @@ standard_parallel : 38.139 cmap : "Spectral_r" # resolution (dots per inch) of output -dpi : 150 - - +dpi : 100