diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index fb0de16910..113ec3f59c 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -30,10 +30,6 @@ jobs: cache-downloads: true cache-environment: true - - name: Checkout externals - run: | - ./manage_externals/checkout_externals ufs-weather-model - - name: Lint the python code run: | micromamba activate srw_app @@ -44,17 +40,22 @@ jobs: pylint ush/set_fv3nml*.py pylint ush/update_input_nml.py + - name: Checkout externals + run: | + ./manage_externals/checkout_externals ufs-weather-model + - name: Run python unittests run: | # exclude test_retrieve_data that is tested in functional test micromamba activate srw_app export UNIT_TEST=True export PYTHONPATH=$(pwd)/ush - python -m unittest -b tests/test_python/*.py + python -m unittest tests/test_python/*.py - name: Run python functional tests run: | micromamba activate srw_app export CI=true export PYTHONPATH=${PWD}/ush - python3 -m unittest -b tests/test_python/test_retrieve_data.py + python3 -m unittest tests/test_python/test_retrieve_data.py + diff --git a/CMakeLists.txt b/CMakeLists.txt index 30e241daba..cf8180097d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,12 +8,20 @@ project(ufs-srweather-app VERSION 1.0 LANGUAGES C CXX Fortran) find_package(MPI REQUIRED COMPONENTS C CXX Fortran) # Set extended version info. -SET(SRWA_VERSION_MAJOR 1) -SET(SRWA_VERSION_MINOR 0) +SET(SRWA_VERSION_MAJOR 2) +SET(SRWA_VERSION_MINOR 2) SET(SRWA_VERSION_PATCH 0) SET(SRWA_VERSION_NOTE "-development") SET(SRWA_VERSION ${SRWA_VERSION_MAJOR}.${SRWA_VERSION_MINOR}.${SRWA_VERSION_PATCH}${SRWA_VERSION_NOTE}) +# Get the latest abbreviated commit hash of the working branch +execute_process( + COMMAND git log -1 --format=%h + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE GIT_HASH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + # A function used to create autotools-style 'yes/no' definitions. # If a variable is set, it 'yes' is returned. Otherwise, 'no' is # returned. @@ -55,7 +63,7 @@ if(NOT DEFINED CMAKE_INSTALL_BINDIR) endif() ##### -# Configure and print the ufs-srweather-app.settings file. +# Configure and print the build settings file. ##### # Determine the configure date. @@ -92,10 +100,12 @@ SET(host_vendor "${osname}") SET(host_os "${osrel}") SET(abs_top_builddir "${CMAKE_CURRENT_BINARY_DIR}") SET(abs_top_srcdir "${CMAKE_CURRENT_SOURCE_DIR}") +SET(application "${APP}") +SET(machine "${BUILD_MACHINE}") SET(CC_VERSION "${CMAKE_C_COMPILER}") -# Set values for .settings file. +# Set values for build settings file. SET(CFLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE}}") SET(CPPFLAGS "${CMAKE_CPP_FLAGS} ${CMAKE_CPP_FLAGS_${CMAKE_BUILD_TYPE}}") SET(LDFLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS_${CMAKE_BUILD_TYPE}}") @@ -107,27 +117,27 @@ is_enabled(BUILD_SHARED_LIBS enable_shared) is_enabled(STATUS_PARALLEL HAS_PARALLEL) # Generate file from template. -CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/ufs_srweather_app.settings.in" - "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app.settings" +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/sorc/build_settings_template.yaml" + "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml" @ONLY) # Read in settings file, print out. # Avoid using system-specific calls so that this # might also work on Windows. -FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app.settings" +FILE(READ "${CMAKE_CURRENT_BINARY_DIR}/build_settings.yaml" UFS-SRWEATHER-APP_SETTINGS) MESSAGE(${UFS-SRWEATHER-APP_SETTINGS}) -# Install ufs_srweather_app.settings file into same location +# Install build settings file into same location # as the app. -INSTALL(FILES "${CMAKE_BINARY_DIR}/ufs_srweather_app.settings" +INSTALL(FILES "${CMAKE_BINARY_DIR}/build_settings.yaml" DESTINATION ${CMAKE_INSTALL_BINDIR}) ##### # Create 'ufs_srweather_app_meta.h' include file. ##### configure_file( - ufs_srweather_app_meta.h.in + sorc/ufs_srweather_app_meta.h.in ufs_srweather_app_meta.h @ONLY) FILE(COPY "${CMAKE_CURRENT_BINARY_DIR}/ufs_srweather_app_meta.h" DESTINATION include) diff --git a/devbuild.sh b/devbuild.sh index 014fbdb3b7..332abb49b5 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -15,8 +15,10 @@ OPTIONS compiler to use; default depends on platform (e.g. intel | gnu | cray | gccgfortran) -a, --app=APPLICATION - weather model application to build; for example, ATMAQ for Online-CMAQ - (e.g. ATM | ATMAQ | ATMW | S2S | S2SW) + weather model application to build; supported SRW options are + ATM (default) Atmosphere only + ATMAQ Online-CMAQ (air quality) + ATMF UFS_FIRE (coupled Community Fire Behavior Model) --ccpp="CCPP_SUITE1,CCPP_SUITE2..." CCPP suites (CCPP_SUITES) to include in build; delimited with ',' --enable-options="OPTION1,OPTION2,..." @@ -363,6 +365,7 @@ fi # cmake settings CMAKE_SETTINGS="\ + -DBUILD_MACHINE=${MACHINE}\ -DCMAKE_BUILD_TYPE=${BUILD_TYPE}\ -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}\ -DCMAKE_INSTALL_BINDIR=${BIN_DIR}\ diff --git a/doc/UsersGuide/BuildingRunningTesting/DefaultVarsTable.rst b/doc/UsersGuide/BuildingRunningTesting/DefaultVarsTable.rst index faaf9129c2..c22751656c 100644 --- a/doc/UsersGuide/BuildingRunningTesting/DefaultVarsTable.rst +++ b/doc/UsersGuide/BuildingRunningTesting/DefaultVarsTable.rst @@ -20,7 +20,7 @@ Table of Variables in ``config_defaults.yaml`` TEST_VX_FCST_INPUT_BASEDIR, FIXgsm, FIXaer, FIXlut, FIXorg, FIXsfc, FIXshp, FIXcrtm, FIXcrtmupp, EXTRN_MDL_DATA_STORES * - Workflow - WORKFLOW_ID, RELATIVE_LINK_FLAG, USE_CRON_TO_RELAUNCH, CRON_RELAUNCH_INTVL_MNTS, CRONTAB_LINE, LOAD_MODULES_RUN_TASK_FP, EXPT_BASEDIR, EXPT_SUBDIR, EXEC_SUBDIR, - EXPTDIR, DOT_OR_USCORE, EXPT_CONFIG_FN, CONSTANTS_FN, RGNL_GRID_NML_FN, FV3_NML_FN, FV3_NML_BASE_SUITE_FN, FV3_NML_YAML_CONFIG_FN, FV3_NML_BASE_ENS_FN, + EXPTDIR, DOT_OR_USCORE, CONSTANTS_FN, RGNL_GRID_NML_FN, FV3_NML_FN, FV3_NML_BASE_SUITE_FN, FV3_NML_YAML_CONFIG_FN, FV3_NML_BASE_ENS_FN, FV3_EXEC_FN, DATA_TABLE_FN, DIAG_TABLE_FN, FIELD_TABLE_FN, DIAG_TABLE_TMPL_FN, FIELD_TABLE_TMPL_FN, MODEL_CONFIG_FN, NEMS_CONFIG_FN, AQM_RC_FN, AQM_RC_TMPL_FN, FV3_NML_BASE_SUITE_FP, FV3_NML_YAML_CONFIG_FP, FV3_NML_BASE_ENS_FP, DATA_TABLE_TMPL_FP, DIAG_TABLE_TMPL_FP, FIELD_TABLE_TMPL_FP, MODEL_CONFIG_TMPL_FP, NEMS_CONFIG_TMPL_FP, AQM_RC_TMPL_FP, DATA_TABLE_FP, FIELD_TABLE_FP, NEMS_CONFIG_FP, FV3_NML_FP, diff --git a/doc/UsersGuide/BuildingRunningTesting/FIRE.rst b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst new file mode 100644 index 0000000000..09a22975a6 --- /dev/null +++ b/doc/UsersGuide/BuildingRunningTesting/FIRE.rst @@ -0,0 +1,257 @@ +.. _UFS_FIRE: + +========================================= +Community Fire Behavior Module (UFS FIRE) +========================================= + +The `Community Fire Behavior Model (CFBM) `_ is a wildland fire model coupled to the UFS Atmospheric Model. The capability to run this code is now available in the UFS Short-Range Weather App for easy use by the community. The `fire_behavior repository `_ is a :term:`submodule` of the UFS Weather Model (WM), coupled through the :term:`NUOPC` Layer to provide direct feedback between the simulated atmosphere and the simulated fire. More information about the CFBM can be found in the :fire-ug:`CFBM Users Guide <>`. + +The biggest difference between the UFS FIRE capability and other modes of the UFS SRW is that a special build flag is required to build the coupled fire behavior code, as described in the instructions below. Aside from that, the need for additional input files, and some fire-specific config settings, configuring and running an experiment is the same as any other use of SRW. + + +.. note:: + + Although this chapter is the primary documentation resource for running the UFS FIRE configuration, users may need to refer to :numref:`Chapter %s ` and :numref:`Chapter %s ` for additional information on building and running the SRW App, respectively. + +Quick Start Guide (UFS FIRE) +===================================== + +Download the Code +------------------- + +Clone the |branch| branch of the authoritative SRW App repository: + +.. code-block:: console + + git clone -b develop https://github.com/ufs-community/ufs-srweather-app + cd ufs-srweather-app + +Checkout Externals +--------------------- + +Users must run the ``checkout_externals`` script to collect (or "check out") the individual components of the SRW App from their respective GitHub repositories. + +.. code-block:: console + + ./manage_externals/checkout_externals + +Build the SRW App with Fire Behavior Enabled +-------------------------------------------- + +To build the SRW with fire behavior code, use the following command: + +.. code-block:: console + + ./devbuild.sh -p= -a=ATMF + +where ```` is ``hera``, ``derecho``, or any other Tier 1 platform. The ``-a`` argument indicates the configuration/version of the application to build; in this case, the atmosphere-fire coupling (ATMF). + +If UFS FIRE builds correctly, users should see the standard executables listed in :numref:`Table %s `. There are no additional files expected, since the CFBM is coupled to the UFS weather model via the same ``ufs_model`` executable. + +Load the |wflow_env| Environment +-------------------------------------------- + +Load the appropriate modules for the workflow: + +.. code-block:: console + + module use /path/to/ufs-srweather-app/modulefiles + module load wflow_ + +where ```` is ``hera``, ``derecho``, or any other Tier 1 platform. + +If the console outputs a message, the user should run the commands specified in the message. For example, if the output says: + +.. code-block:: console + + Please do the following to activate conda: + > conda activate srw_app + +then the user should run |activate|. Otherwise, the user can continue with configuring the workflow. + +.. _FIREConfig: + +Configure Experiment +--------------------------- + +Users will need to configure their experiment by setting parameters in the ``config.yaml`` file. To start, users can copy an example experiment setting into ``config.yaml``: + +.. code-block:: console + + cd ush + cp config.fire.yaml config.yaml + +Users will need to change the ``MACHINE`` and ``ACCOUNT`` variables in ``config.yaml`` to match their system. They may also wish to adjust other experiment settings, especially under the ``fire:`` section, described in further detail below. For more information on other configuration settings, see :numref:`Section %s `. + +Activating the fire behavior module is done by setting ``UFS_FIRE: True`` in the ``fire:`` section of your ``config.yaml`` file. If this variable is not specified or set to false, a normal atmospheric simulation will be run, without fire settings. + +.. code-block:: console + + fire: + UFS_FIRE: True + +The fire module has the ability to print out additional messages to the log file for debugging; to enable additional log output (which may slow down the integration considerably, especially at higher levels) set ``FIRE_PRINT_MSG`` > 0 + +.. code-block:: console + + fire: + FIRE_PRINT_MSG: 1 + +Additional boundary conditions file +----------------------------------- +The CFBM, as an independent, coupled component, runs separately from the atmospheric component of the weather model, requires an additional input file (``geo_em.d01.nc``) that contains fire-specific boundary conditions such as fuel properties. On Level 1 systems, users can find an example file in the usual :ref:`input data locations ` under ``LOCATION``. Users can also download the data required for the community experiment from the `UFS SRW App Data Bucket `__. + + +Instructions on how to create this file for your own experiment can be found in the :fire-ug:`CFBM Users Guide `. + +Once the file is acquired/created, you will need to specify its location in your ``config.yaml`` file with the setting ``FIRE_INPUT_DIR``. + +.. code-block:: console + + fire: + FIRE_INPUT_DIR: /directory/containing/geo_em/file + + + +Specifying a fire ignition +--------------------------- + +The CFBM simulates fires by specifying an "ignition" that will then propogate based on the atmospheric conditions and the specified settings. An ignition can either be a "point ignition" (i.e. a disk of fire some specified radius around a single location), or a straight line linear ignition specified by a start and end location and a specified "radius" (width). The ignition can start at the beginning of your simulation, or at some time later as specified. The CFBM can support up to 5 different fire ignitions at different places and times in a given simulation. + +The CFBM settings are controlled by the :term:`namelist` file ``namelist.fire``. The available settings in this file are described in the :fire-ug:`CFBM Users Guide `, and an example file can be found under ``parm/namelist.fire``. However, there is no need to manually provide or edit this file, as the SRW workflow will create the fire namelist using the user settings in ``config.yaml``. The fire-specific options in SRW are documented in :numref:`Section %s `. + +Example fire configuration +--------------------------- + +Here is one example of settings that can be specified for a UFS FIRE simulation: + +.. code-block:: console + + fire: + UFS_FIRE: True + FIRE_INPUT_DIR: /home/fire_input + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 1800 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 + +In this case, a single fire (``FIRE_NUM_IGNITIONS: 1``) of radius 250 meters (``FIRE_IGNITION_RADIUS: 250``) is ignited at latitude 40.609˚N (``FIRE_IGNITION_START_LAT: 40.609``), 105.879˚W (``FIRE_IGNITION_START_LON: -105.879``) 6480 seconds after the start of the simulation (``FIRE_IGNITION_START_TIME: 6480``) with a rate of spread specified as 0.05 m/s (``FIRE_IGNITION_ROS: 0.05``). This "ignition" ends 7000 seconds after the start of the simulation (``FIRE_IGNITION_END_TIME: 7000``), after which the fire behavior is completely governed by the physics of the fire behavior model (integrated every 0.5 seconds as specified by ``OUTPUT_DT_FIRE``), the input fuel conditions, and the simulated atmospheric conditions. + +The CFBM creates output files in :term:`netCDF` format, with the naming scheme ``fire_output_YYYY-MM-DD_hh:mm:ss.nc``. In this case the output files are written every 30 minutes (``OUTPUT_DT_FIRE: 1800``). + +.. note:: + + Any of the settings under :fire-ug:`the &fire section of the namelist ` can be specified in the SRW App ``config.yaml`` file under the ``fire:`` section, not just the settings described above. However, any additional settings from ``namelist.fire`` will need to be added to ``config_defaults.yaml`` first; otherwise the check for valid SRW options will fail. + +To specify multiple fire ignitions (``FIRE_NUM_IGNITIONS > 1``), the above settings will need to be specified as a list, with one entry per ignition. See :numref:`Section %s ` for more details. + + +Generate the Workflow +------------------------ + +Generate the workflow: + +.. code-block:: console + + ./generate_FV3LAM_wflow.py + +Run the Workflow +------------------ + +If ``USE_CRON_TO_RELAUNCH`` is set to true in ``config.yaml``, the workflow will run automatically. If it was set to false, users must submit the workflow manually from the experiment directory: + +.. code-block:: console + + cd ${EXPT_BASEDIR}/${EXPT_SUBDIR} + ./launch_FV3LAM_wflow.sh + +Repeat the launch command regularly until a SUCCESS or FAILURE message appears on the terminal window. See :numref:`Section %s ` for more on the ``${EXPT_BASEDIR}`` and ``${EXPT_SUBDIR}`` variables. + +Users may check experiment status from the experiment directory with either of the following commands: + +.. code-block:: console + + # Check the experiment status (for cron jobs) + rocotostat -w FV3LAM_wflow.xml -d FV3LAM_wflow.db -v 10 + + # Check the experiment status and relaunch the workflow (for manual jobs) + ./launch_FV3LAM_wflow.sh; tail -n 40 log.launch_FV3LAM_wflow + +.. _FIRESuccess: + +Experiment Output +-------------------- + +The workflow run is complete when all tasks display a "SUCCEEDED" message. If everything goes smoothly, users will eventually see a workflow status table similar to the following: + +.. code-block:: console + + CYCLE TASK JOBID STATE EXIT STATUS TRIES DURATION + ================================================================================================================================ + 202008131800 make_grid 6498125 SUCCEEDED 0 1 70.0 + 202008131800 make_orog 6498145 SUCCEEDED 0 1 87.0 + 202008131800 make_sfc_climo 6498172 SUCCEEDED 0 1 90.0 + 202008131800 get_extrn_ics 6498126 SUCCEEDED 0 1 46.0 + 202008131800 get_extrn_lbcs 6498127 SUCCEEDED 0 1 46.0 + 202008131800 make_ics_mem000 6498202 SUCCEEDED 0 1 91.0 + 202008131800 make_lbcs_mem000 6498203 SUCCEEDED 0 1 106.0 + 202008131800 run_fcst_mem000 6498309 SUCCEEDED 0 1 1032.0 + 202008131800 run_post_mem000_f000 6498336 SUCCEEDED 0 1 75.0 + 202008131800 run_post_mem000_f001 6498387 SUCCEEDED 0 1 76.0 + 202008131800 run_post_mem000_f002 6498408 SUCCEEDED 0 1 75.0 + 202008131800 run_post_mem000_f003 6498409 SUCCEEDED 0 1 75.0 + 202008131800 run_post_mem000_f004 6498432 SUCCEEDED 0 1 64.0 + 202008131800 run_post_mem000_f005 6498433 SUCCEEDED 0 1 77.0 + 202008131800 run_post_mem000_f006 6498435 SUCCEEDED 0 1 74.0 + 202008131800 integration_test_mem000 6498434 SUCCEEDED 0 1 27.0 + +In addition to the standard UFS and UPP output described elsewhere in this users guide, the UFS_FIRE runs produce additional output files :ref:`described above `: + +.. code-block:: console + + $ cd /path/to/expt_dir/experiment + $ ls 2020081318/fire_output* + fire_output_2020-08-13_18:00:00.nc fire_output_2020-08-13_19:30:00.nc fire_output_2020-08-13_21:00:00.nc fire_output_2020-08-13_22:30:00.nc + fire_output_2020-08-13_18:30:00.nc fire_output_2020-08-13_20:00:00.nc fire_output_2020-08-13_21:30:00.nc fire_output_2020-08-13_23:00:00.nc + fire_output_2020-08-13_19:00:00.nc fire_output_2020-08-13_20:30:00.nc fire_output_2020-08-13_22:00:00.nc fire_output_2020-08-13_23:30:00.nc + +These files contain output directly from the fire model (hence why they are at a greater frequency), including variables such as the fire perimeter and area, smoke emitted, and fuel percentage burnt. + +.. image:: https://github.com/ufs-community/ufs-srweather-app/wiki/FIRE/ncview.emis_smoke_trim.png + :alt: Image of the simulated fire area from an example run + :align: center + +.. _FIRE-WE2E: + + +WE2E Tests for FIRE +======================= + +Build the app for FIRE: + +.. code-block:: console + + ./devbuild.sh -p=hera -a=ATMF + + +Run the WE2E tests: + +.. code-block:: console + + $ cd /path/to/ufs-srweather-app/tests/WE2E + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t fire + +You can also run each test individually if needed: + + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t UFS_FIRE_one-way-coupled + $ ./run_WE2E_tests.py -t my_tests.txt -m hera -a gsd-fv3 -q -t UFS_FIRE_multifire_one-way-coupled + + + diff --git a/doc/UsersGuide/BuildingRunningTesting/index.rst b/doc/UsersGuide/BuildingRunningTesting/index.rst index 5c0efc3c64..a0aa69c85b 100644 --- a/doc/UsersGuide/BuildingRunningTesting/index.rst +++ b/doc/UsersGuide/BuildingRunningTesting/index.rst @@ -13,3 +13,4 @@ Building, Running, and Testing the SRW App Tutorial VXCases AQM + FIRE diff --git a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst index d9704b1ab6..8b9e7648a7 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/ConfigWorkflow.rst @@ -112,6 +112,7 @@ If non-default parameters are selected for the variables in this section, they s ``SCHED``: (Default: "") The job scheduler to use (e.g., Slurm) on the specified ``MACHINE``. Leaving this an empty string allows the experiment generation script to set it automatically depending on the machine the workflow is running on. Valid values: ``"slurm"`` | ``"pbspro"`` | ``"lsf"`` | ``"lsfcray"`` | ``"none"`` + Machine-Dependent Parameters ------------------------------- These parameters vary depending on machine. On :srw-wiki:`Level 1 and 2 ` systems, the appropriate values for each machine can be viewed in the ``ush/machine/.sh`` scripts. To specify a value other than the default, add these variables and the desired value in the ``config.yaml`` file so that they override the ``config_defaults.yaml`` and machine default values. @@ -344,9 +345,6 @@ Pre-Processing File Separator Parameters Set File Name Parameters ---------------------------- -``EXPT_CONFIG_FN``: (Default: "config.yaml") - Name of the user-specified configuration file for the forecast experiment. - ``CONSTANTS_FN``: (Default: "constants.yaml") Name of the file containing definitions of various mathematical, physical, and SRW App constants. @@ -1029,9 +1027,15 @@ FVCOM Parameters ``FVCOM_FILE``: (Default: "fvcom.nc") Name of the file located in ``FVCOM_DIR`` that has :term:`FVCOM` data interpolated to the FV3-LAM grid. This file will be copied later to a new location, and the name will be changed to ``fvcom.nc`` if a name other than ``fvcom.nc`` is selected. -Vertical Coordinate File Parameter +Vertical Coordinate Parameters ------------------------------------ +``LEVP``: (Default: 65) + Number of vertical levels in the atmosphere. In order to change this + number, the user will additionally need to create a vertical coordinate + distribution file; this process is described in :numref:`Section %s ` + This value should be the same in both ``make_ics`` and ``make_lbcs``. + ``VCOORD_FILE``: (Default: ``"{{ workflow.FIXam }}/global_hyblev.l65.txt"``) Full path to the file used to set the vertical coordinates in FV3. This file should be the same in both ``make_ics`` and ``make_lbcs``. @@ -1049,9 +1053,15 @@ Non-default parameters for the ``make_lbcs`` task are set in the ``task_make_lbc ``OMP_STACKSIZE_MAKE_LBCS``: (Default: "1024m") Controls the size of the stack for threads created by the OpenMP implementation. -Vertical Coordinate File Parameter +Vertical Coordinate Parameters ------------------------------------ +``LEVP``: (Default: 65) + Number of vertical levels in the atmosphere. In order to change this + number, the user will additionally need to create a vertical coordinate + distribution file; this process is described in :numref:`Section %s ` + This value should be the same in both ``make_ics`` and ``make_lbcs``. + ``VCOORD_FILE``: (Default: ``"{{ workflow.FIXam }}/global_hyblev.l65.txt"``) Full path to the file used to set the vertical coordinates in FV3. This file should be the same in both ``make_ics`` and ``make_lbcs``. @@ -1809,6 +1819,117 @@ Non-default parameters for coupled Air Quality Modeling (AQM) tasks are set in t ``NEXUS_GFS_SFC_ARCHV_DIR``: (Default: "/NCEPPROD/hpssprod/runhistory") Path to archive directory for gfs surface files on HPSS. +.. _fire-parameters: + +Community Fire Behavior Model Parameters +======================================== + +Non-default parameters for the Community Fire Behavior Model (CFBM) in SRW are set in the ``fire:`` section of the ``config.yaml`` file. + +``UFS_FIRE``: (Default: false) + Flag for turning on CFBM fire simulation + +``FIRE_INPUT_DIR``: (Default: "") + Directory where fire input file (geo_em.d01.nc) can be found + +``DT_FIRE``: (Default: 0.5) + The fire behavior component’s integration timestep in seconds + +``OUTPUT_DT_FIRE``: (Default: 300) + The fire behavior component’s output timestep in seconds + +``FIRE_NUM_TASKS``: (Default: 0) + Number of MPI tasks assigned to the FIRE_BEHAVIOR component. Currently only 1 task is supported. + +.. note:: + The following options control namelist values in the ``&fire`` section of the Community Fire + Behavior Model. See the :fire-ug:`CFBM Users Guide ` for more information. + +``FIRE_PRINT_MSG``: (Default: 0) + Debug print level for the weather model fire core. Levels greater than 1 will print extra + messages to the log file at run time. + + 0: no extra prints + + 1: Extra prints + + 2: More extra prints + + 3: Even more extra prints + +``FIRE_WIND_HEIGHT``: (Default: 5.0) + Height to interpolate winds to for calculating fire spread rate + +``FIRE_ATM_FEEDBACK``: (Default: 0.0) + Multiplier for heat fluxes. Use 1.0 for normal two-way coupling. Use 0.0 for one-way coupling. + Intermediate values will vary the amount of forcing provided from the fire to the dynamical core. + +``FIRE_VISCOSITY``: (Default: 0.4) + Artificial viscosity in level set method. Maximum value of 1. Required for ``FIRE_UPWINDING=0`` + +``FIRE_UPWINDING``: (Default: 9) + Upwinding scheme used for calculating the normal spread of the fire front. More detailed descriptions + of these options can be found in the :fire-ug:`CFBM Users Guide `. + + 0 = Central Difference + + 1 = Standard + + 2 = Godunov + + 3 = ENO1 + + 4 = Sethian + + 5 = 2nd-order Sethian + + 6 = WENO3 + + 7 = WENO5 + + 8 = Hybrid WENO3/ENO1 + + 9 = Hybrid WENO5/ENO1 + +``FIRE_LSM_ZCOUPLING`` (Default: false) + When true, uses ``FIRE_LSM_ZCOUPLING_REF`` instead of ``FIRE_WIND_HEIGHT`` as a reference height + to calculate the logarithmic surface layer wind profile + +``FIRE_LSM_ZCOUPLING_REF`` (Default: 60.0) + Reference height from which the velocity at FIRE_WIND_HEIGHT is calculated using a logarithmic profile + + +``FIRE_NUM_IGNITIONS`` (Default: 1) + Number of fire ignitions. + +.. note:: + If ``FIRE_NUM_IGNITIONS > 1``, the following variables should be lists with one entry for each ignition + +``FIRE_IGNITION_ROS`` (Default: 0.0) + Ignition rate of spread (Rothermel parameterization) + +``FIRE_IGNITION_START_LAT`` (Default: 40.609) + Latitude for start of ignition(s) + +``FIRE_IGNITION_START_LON`` (Default: -105.879) + Longitude for start of ignition(s) + +``FIRE_IGNITION_END_LAT`` (Default: 40.609) + Latitude for end of ignition(s) + +``FIRE_IGNITION_END_LON`` (Default: -105.879) + Longitude for end of ignition(s) + +``FIRE_IGNITION_RADIUS`` (Default: 250) + Radius of ignition area in meters + +``FIRE_IGNITION_START_TIME`` (Default: 6480) + Start time of ignition(s) in seconds (counting from the beginning of the simulation) + +``FIRE_IGNITION_START_TIME`` (Default: 7000) + End time of ignition(s) in seconds (counting from the beginning of the simulation) + + Rocoto Parameters =================== diff --git a/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst b/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst index 482caf8590..7bc13c0794 100644 --- a/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst +++ b/doc/UsersGuide/CustomizingTheWorkflow/LAMGrids.rst @@ -429,37 +429,24 @@ After hitting ``Enter``, the program will print a ``pmin`` value (e.g., ``pmin= Configure the SRW App ----------------------- -To use the new ``ak``/``bk`` file to define vertical levels in an experiment, users will need to modify the input namelist file (``input.nml.FV3``) and their configuration file (``config.yaml``). - -Modify ``input.nml.FV3`` -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The FV3 namelist file, ``input.nml.FV3``, is located in ``ufs-srweather-app/parm``. Users will need to update the ``levp`` and ``npz`` variables in this file. For ``n`` vertical levels, users should set ``levp=n`` and ``npz=n-1``. For example, a user who wants 128 vertical levels would set ``levp`` and ``npz`` as follows: - -.. code-block:: console - - &external_ic_nml - levp = 128 - - &fv_core_nml - npz = 127 - -Additionally, check that ``external_eta = .true.``. - -.. note:: - - Keep in mind that levels and layers are not the same. In UFS code, ``levp`` is the number of vertical *levels*, and ``npz`` is the number of vertical levels without TOA. Thus, ``npz`` is equivalent to the number of vertical *layers*. For ``v`` vertical *layers*, set ``npz=v`` and ``levp=v+1``. Use the value of ``levp`` as the number of vertical levels when generating ``ak``/``bk``. +To use the new ``ak``/``bk`` file to define vertical levels in an experiment, users will need to modify their configuration file (``config.yaml``). Modify ``config.yaml`` ^^^^^^^^^^^^^^^^^^^^^^^^ -To use the text file produced by ``vcoord_gen`` in the SRW App, users need to set the ``VCOORD_FILE`` variable in their ``config.yaml`` file. Normally, this file is named ``global_hyblev.l65.txt`` and is located in the ``fix_am`` directory on Level 1 systems, but users should adjust the path and name of the file to suit their system. For example, in ``config.yaml``, a user (Jane Smith) might set: +To use the new vertical levels produced by ``vcoord_gen`` in the SRW App, users need to set the ``VCOORD_FILE`` and ``LEVP`` variables in their ``config.yaml`` file. Normally, the vertical coordinate file is named ``global_hyblev.l65.txt`` and is located in the ``fix_am`` directory on Level 1 systems, but users should adjust the path and name of the file to suit their system. Additionally, update the ``LEVP`` variable for the new number of vertical levels. For example, in ``config.yaml``, a user (Jane Smith) might set: .. code-block:: console task_make_ics: + LEVP: 65 VCOORD_FILE: /Users/Jane.Smith/ufs-srweather-app/parm/global_hyblev.L128.txt task_make_lbcs: + LEVP: 65 VCOORD_FILE: /Users/Jane.Smith/ufs-srweather-app/parm/global_hyblev.L128.txt +.. note:: + + Keep in mind that levels and layers are not the same. In UFS code, ``levp`` is the number of vertical *levels*, and ``npz`` is the number of vertical levels without TOA. Thus, ``npz`` in the UFS namelist file is equivalent to the number of vertical *layers*. For ``v`` vertical *layers*, ``npz=v`` and ``levp=v+1``. Use the value of ``levp`` as the number of vertical levels when generating ``ak``/``bk``. + Configure other variables as desired and generate the experiment as described in :numref:`Section %s `. diff --git a/doc/UsersGuide/Reference/Glossary.rst b/doc/UsersGuide/Reference/Glossary.rst index 7ffc569b21..48fb970cab 100644 --- a/doc/UsersGuide/Reference/Glossary.rst +++ b/doc/UsersGuide/Reference/Glossary.rst @@ -239,6 +239,9 @@ Glossary spack-stack The `spack-stack `_ is a collaborative effort between the NOAA Environmental Modeling Center (EMC), the UCAR Joint Center for Satellite Data Assimilation (JCSDA), and the Earth Prediction Innovation Center (EPIC). *spack-stack* is a repository that provides a :term:`Spack`-based method for building the software stack required for numerical weather prediction (NWP) tools such as the `Unified Forecast System (UFS) `_ and the `Joint Effort for Data assimilation Integration (JEDI) `_ framework. *spack-stack* uses the Spack package manager along with custom Spack configuration files and Python scripts to simplify installation of the libraries required to run various applications. The *spack-stack* can be installed on a range of platforms and comes pre-configured for many systems. Users can install the necessary packages for a particular application and later add the missing packages for another application without having to rebuild the entire stack. To get started, check out the documentation :doc:`here `. + submodule + A `submodule `_ is a git repository linked to another repository as a subdirectory. Many UFS components are linked in this way; for example, the :term:`UPP` repository is a submodule of the :term:`FV3` repository. + tracer According to the American Meteorological Society (AMS) definition, a `tracer `_ is "Any substance in the atmosphere that can be used to track the history [i.e., movement] of an air mass." Tracers are carried around by the motion of the atmosphere (i.e., by :term:`advection`). These substances are usually gases (e.g., water vapor, CO2), but they can also be non-gaseous (e.g., rain drops in microphysics parameterizations). In weather models, temperature (or potential temperature), absolute humidity, and radioactivity are also usually treated as tracers. According to AMS, "The main requirement for a tracer is that its lifetime be substantially longer than the transport process under study." diff --git a/doc/conf.py b/doc/conf.py index 5989ae0d74..2b5bf7b4d4 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -307,5 +307,6 @@ def warn_undocumented_members(app, what, name, obj, options, lines): 'srw-repo': ('https://github.com/ufs-community/ufs-srweather-app/%s', '%s'), 'srw-wiki': ('https://github.com/ufs-community/ufs-srweather-app/wiki/%s','%s'), 'uw': ('https://uwtools.readthedocs.io/en/main/%s', '%s'), + 'fire-ug': ('https://fire-behavior.readthedocs.io/en/latest/%s', '%s'), } diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index d3d20a5bb5..d7ef51b1e7 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -28,6 +28,8 @@ load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.9.0.1")) load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index e01b68905e..54c82569e9 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -21,6 +21,8 @@ load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) setenv("CFLAGS","-diag-disable=10441") setenv("FFLAGS","-diag-disable=10441") +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_jet_intel.lua b/modulefiles/build_jet_intel.lua index 78e70e0960..04124e4bf5 100644 --- a/modulefiles/build_jet_intel.lua +++ b/modulefiles/build_jet_intel.lua @@ -17,6 +17,8 @@ load("srw_common") load("nccmp/1.9.0.1") load("nco/5.0.6") +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_odin_intel.lua b/modulefiles/build_odin_intel.lua index f4059a868a..40c3ada501 100644 --- a/modulefiles/build_odin_intel.lua +++ b/modulefiles/build_odin_intel.lua @@ -45,6 +45,8 @@ load("sigio") load("w3emc") load("wgrib2") +setenv("FC", "ftn") + setenv("CMAKE_C_COMPILER","cc") setenv("CMAKE_CXX_COMPILER","CC") setenv("CMAKE_Fortran_COMPILER","ftn") diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index 92d268a4d7..31efcb57c6 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -21,6 +21,8 @@ load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) setenv("CFLAGS","-diag-disable=10441") setenv("FFLAGS","-diag-disable=10441") +setenv("FC", "mpiifort") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") diff --git a/modulefiles/build_singularity_gnu.lua b/modulefiles/build_singularity_gnu.lua index 434ac448d4..275842ffc2 100644 --- a/modulefiles/build_singularity_gnu.lua +++ b/modulefiles/build_singularity_gnu.lua @@ -33,6 +33,8 @@ load("gfsio") load("wgrib2") load("upp") +setenv("FC", "mpif90") + setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpicxx") setenv("CMAKE_Fortran_COMPILER","mpif90") diff --git a/parm/diag_table.FV3_HRRR b/parm/diag_table.FV3_HRRR index 893ca25a2b..c7bbc59b9e 100755 --- a/parm/diag_table.FV3_HRRR +++ b/parm/diag_table.FV3_HRRR @@ -339,6 +339,9 @@ "gfs_phys", "spp_wts_rad", "spp_wts_rad", "fv3_history", "all", .false., "none", 2 "gfs_phys", "spp_wts_gwd", "spp_wts_gwd", "fv3_history", "all", .false., "none", 2 +# Additional entries from user-specified SRW settings (e.g. UFS-FIRE) +{{ additional_entries }} + #============================================================================================= # #====> This file can be used with diag_manager/v2.0a (or higher) <==== diff --git a/parm/namelist.fire b/parm/namelist.fire new file mode 100644 index 0000000000..4ef039996b --- /dev/null +++ b/parm/namelist.fire @@ -0,0 +1,28 @@ + &time + dt = 0.5 + interval_output = 300 +/ + + &atm + interval_atm = 1 + kde = 65 + / + + &fire + fire_num_ignitions = 1, + fire_ignition_ros1 = 0.05, + fire_ignition_start_lat1 = 40.609, + fire_ignition_start_lon1 = -105.879, + fire_ignition_end_lat1 = 40.609, + fire_ignition_end_lon1 = -105.879, + fire_ignition_radius1 = 250, + fire_ignition_start_time1 = 6480, + fire_ignition_end_time1 = 7000, + fire_wind_height = 5.0, ! height to interpolate winds to for calculating fire spread rate + fire_print_msg = 1, ! 1 print fire debugging messages + fire_atm_feedback = 1.0, ! real, multiplier for heat fluxes, 1.=normal, 0.=turn off two-way coupling + fire_viscosity = 0.4, ! artificial viscosity in level set method (max 1, needed with fire_upwinding=0) + fire_upwinding = 9, ! 0=none, 1=standard, 2=godunov, 3=eno, 4=sethian + fire_lsm_zcoupling = .false., + fire_lsm_zcoupling_ref = 60.0, +/ diff --git a/parm/ufs.configure b/parm/ufs.configure index 48d2a66e8a..74bd602231 100644 --- a/parm/ufs.configure +++ b/parm/ufs.configure @@ -3,63 +3,47 @@ ############################################# # ESMF # -{%- if print_esmf %} -logKindFlag: ESMF_LOGKIND_MULTI -{%- else %} -logKindFlag: ESMF_LOGKIND_MULTI_ON_ERROR -{%- endif %} +logKindFlag: {{ logKindFlag }} globalResourceControl: true -{% if cpl_aqm %} # EARTH # -EARTH_component_list: ATM AQM +EARTH_component_list: {{ EARTH_cl }} EARTH_attributes:: Verbosity = 0 :: # ATM # ATM_model: fv3 -ATM_petlist_bounds: 0 {{ pe_member01_m1 }} +ATM_petlist_bounds: {{ ATM_pb }}{{ +ATM_omp_num_threads_line }} ATM_attributes:: - Verbosity = 0 + Verbosity = 0{{ +ATM_diag_line }} :: +{% if cpl_aqm %} # AQM # AQM_model: aqm -AQM_petlist_bounds: 0 {{ aqm_pe_member01_m1 }} +AQM_petlist_bounds: {{ AQM_pb }} AQM_attributes:: Verbosity = 0 Diagnostic = 0 :: - -# Run Sequence # -runSeq:: - @{{ dt_atmos }} - ATM phase1 - ATM -> AQM - AQM - AQM -> ATM - ATM phase2 - @ -:: -{% else %} -# EARTH # -EARTH_component_list: ATM -EARTH_attributes:: - Verbosity = 0 -:: - -# ATM # -ATM_model: fv3 -ATM_petlist_bounds: 0 {{ pe_member01_m1 }} -ATM_omp_num_threads: {{ atm_omp_num_threads }} -ATM_attributes:: - Verbosity = 0 +{% endif %} +{% if ufs_fire %} +# FIRE # +FIRE_model: fire_behavior +FIRE_petlist_bounds: {{ FIRE_pb }} +FIRE_attributes:: + Verbosity = high Diagnostic = 0 :: +{% endif %} # Run Sequence # runSeq:: - ATM +{% for line in runseq -%} +{{ line }} +{%- endfor %} :: -{% endif %} + diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 0241dbd728..c5e7f6601e 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -117,7 +117,7 @@ # . $USHdir/source_util_funcs.sh for sect in user nco platform workflow global cpl_aqm_parm constants fixed_files \ - task_get_extrn_lbcs task_run_fcst task_run_post ; do + task_get_extrn_lbcs task_run_fcst task_run_post fire; do source_yaml ${GLOBAL_VAR_DEFNS_FP} ${sect} done @@ -196,7 +196,6 @@ if [ ${#FCST_LEN_CYCL[@]} -gt 1 ]; then CYCLE_IDX=$(( ${cyc_mod} / ${INCR_CYCL_FREQ} )) FCST_LEN_HRS=${FCST_LEN_CYCL[$CYCLE_IDX]} fi - # #----------------------------------------------------------------------- # @@ -756,6 +755,49 @@ fi # #----------------------------------------------------------------------- # +# Call the function for updating the &time section of namelist.fire +# +#----------------------------------------------------------------------- +# +if [ $(boolify "${UFS_FIRE}") = "TRUE" ]; then + FCST_END_DATE=$( $DATE_UTIL --utc --date "${PDY} ${cyc} UTC + ${FCST_LEN_HRS} hours" "+%Y%m%d%H%M%S" ) + # This horrible syntax $((10#$VARNAME)) is to force bash to treat numbers as decimal instead of + # trying to octal all up in our business + settings=" + &time + start_year = $((10#${CDATE:0:4})), + start_month = $((10#${CDATE:4:2})), + start_day = $((10#${CDATE:6:2})), + start_hour = $((10#${CDATE:8:2})), + start_minute = 00, + start_second = 00, + end_year = $((10#${FCST_END_DATE:0:4})), + end_month = $((10#${FCST_END_DATE:4:2})), + end_day = $((10#${FCST_END_DATE:6:2})), + end_hour = $((10#${FCST_END_DATE:8:2})), + end_minute = $((10#${FCST_END_DATE:10:2})), + end_second = $((10#${FCST_END_DATE:12:2})), + / +" + + echo $settings | uw config realize --update-format nml --input-format nml --output-format nml --input-file "${FIRE_NML_FP}" -o "${FIRE_NML_FN}" + err=$? + if [ $err -ne 0 ]; then + print_err_msg_exit "\ + Call to uw config realize to create ${FIRE_NML_FN} failed. + Parameters passed to this script are: + FIRE_NML_FN = \"${FIRE_NML_FN}\" + FIRE_NML_FP = \"${FIRE_NML_FP}\" + Namelist settings specified on command line: + settings = +$settings" + fi + # Link fire input file + create_symlink_to_file ${FIRE_INPUT_DIR}/geo_em.d01.nc geo_em.d01.nc FALSE +fi +# +#----------------------------------------------------------------------- +# # Call the function that creates the diag_table file within each cycle # directory. # diff --git a/sorc/build_settings_template.yaml b/sorc/build_settings_template.yaml new file mode 100644 index 0000000000..792ee540c4 --- /dev/null +++ b/sorc/build_settings_template.yaml @@ -0,0 +1,23 @@ +# UFS Short Range Weather App Configuration Summary +#=================================================== + +# General +#--------- +SRW_Version: @SRWA_VERSION@ +SRW_hash: @GIT_HASH@ +Machine: @machine@ +Configured_On: @CONFIG_DATE@ +Host_System: @host_cpu@-@host_vendor@-@host_os@ +Build_Directory: @abs_top_builddir@ +Install_Prefix: @prefix@ +Application: @application@ +# Compiling Options +#------------------- +C_Compiler: @CC_VERSION@ +CFLAGS: @CFLAGS@ +CPPFLAGS: @CPPFLAGS@ +LDFLAGS: @LDFLAGS@ +Shared_Library: @enable_shared@ +Static_Library: @enable_static@ +Extra_libraries: @LIBS@ + diff --git a/ufs_srweather_app_meta.h.in b/sorc/ufs_srweather_app_meta.h.in similarity index 100% rename from ufs_srweather_app_meta.h.in rename to sorc/ufs_srweather_app_meta.h.in diff --git a/tests/WE2E/WE2E_summary.py b/tests/WE2E/WE2E_summary.py index 16b94aaf3b..147380eb4e 100755 --- a/tests/WE2E/WE2E_summary.py +++ b/tests/WE2E/WE2E_summary.py @@ -6,7 +6,13 @@ sys.path.append("../../ush") -from python_utils import load_config_file +try: + from python_utils import load_config_file +except ModuleNotFoundError: + print("\n\nERROR: Could not load python utilities.") + print('Note that this script can only be run in the SRW App from the directory:') + print("ufs-srweather-app/tests/WE2E\n\n") + raise from check_python_version import check_python_version @@ -68,9 +74,13 @@ def setup_logging(debug: bool = False) -> None: else: raise ValueError(f'Bad arguments; run {__file__} -h for more information') - # Calculate core hours and update yaml - expts_dict = calculate_core_hours(expts_dict) - write_monitor_file(yaml_file,expts_dict) + if expts_dict: + # Calculate core hours and update yaml + expts_dict = calculate_core_hours(expts_dict) + write_monitor_file(yaml_file,expts_dict) + + #Call function to print summary + print_WE2E_summary(expts_dict, args.debug) + else: + logging.error(f'No experiments found in provided directory {args.expt_dir}') - #Call function to print summary - print_WE2E_summary(expts_dict, args.debug) diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index d6d0c016e6..78655857e8 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -230,7 +230,7 @@ def run_we2e_tests(homedir, args) -> None: if args.quiet: console_handler = logging.getLogger().handlers[1] console_handler.setLevel(logging.WARNING) - expt_dir = generate_FV3LAM_wflow(ushdir,logfile=f"{ushdir}/log.generate_FV3LAM_wflow", + expt_dir = generate_FV3LAM_wflow(ushdir,"config.yaml",logfile=f"{ushdir}/log.generate_FV3LAM_wflow", debug=args.debug) if args.quiet: if args.debug: diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml new file mode 100644 index 0000000000..6bd217914f --- /dev/null +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_multifire_one-way-coupled.yaml @@ -0,0 +1,76 @@ +metadata: + description: >- + Tests the UFS_FIRE capability for multiple fire ignitions with one-way (ATM-->FIRE) coupling +user: + RUN_ENVIR: community +workflow: + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: SUBCONUS_CO_3km + DATE_FIRST_CYCL: '2020081318' + DATE_LAST_CYCL: '2020081318' + FCST_LEN_HRS: 3 # 27 hours of LBCs staged on Derecho + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_NAME_ICS: HRRR + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' +task_get_extrn_lbcs: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_FILES_LBCS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' + EXTRN_MDL_NAME_LBCS: HRRR + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: grib2 +task_run_fcst: + QUILTING: true + PRINT_ESMF: True + OMP_NUM_THREADS_RUN_FCST: 1 +task_plot_allvars: + COMOUT_REF: "" +fire: + UFS_FIRE: True + FIRE_NUM_TASKS: 1 + FIRE_INPUT_DIR: '{{ platform.WE2E_TEST_DATA }}/UFS_FIRE/{{ workflow.DATE_FIRST_CYCL }}' + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_IGNITIONS: 2 + FIRE_IGNITION_ROS: + - 0.05 + - 0.1 + FIRE_IGNITION_START_LAT: + - 40.609 + - 40.666 + FIRE_IGNITION_START_LON: + - -105.879 + - -105.95 + FIRE_IGNITION_END_LAT: + - 40.609 + - 40.678 + FIRE_IGNITION_END_LON: + - -105.879 + - -105.94 + FIRE_IGNITION_RADIUS: + - 250 + - 100 + FIRE_IGNITION_START_TIME: + - 6480 + - 3600 + FIRE_IGNITION_END_TIME: + - 7000 + - 3720 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_ATM_FEEDBACK: 0.0 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 01:00:00 + diff --git a/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml new file mode 100644 index 0000000000..756dac8135 --- /dev/null +++ b/tests/WE2E/test_configs/fire/config.UFS_FIRE_one-way-coupled.yaml @@ -0,0 +1,60 @@ +metadata: + description: >- + Tests the UFS_FIRE capability for a single fire with one-way (ATM-->FIRE) coupling +user: + RUN_ENVIR: community +workflow: + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: SUBCONUS_CO_3km + DATE_FIRST_CYCL: '2020081318' + DATE_LAST_CYCL: '2020081318' + FCST_LEN_HRS: 3 # 27 hours of LBCs staged on Derecho + PREEXISTING_DIR_METHOD: rename +task_get_extrn_ics: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_NAME_ICS: HRRR + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' +task_get_extrn_lbcs: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_FILES_LBCS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' + EXTRN_MDL_NAME_LBCS: HRRR + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: grib2 +task_run_fcst: + QUILTING: true + PRINT_ESMF: True + OMP_NUM_THREADS_RUN_FCST: 1 +task_plot_allvars: + COMOUT_REF: "" +fire: + UFS_FIRE: True + FIRE_NUM_TASKS: 1 + FIRE_INPUT_DIR: '{{ platform.WE2E_TEST_DATA }}/UFS_FIRE/{{ workflow.DATE_FIRST_CYCL }}' + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_ATM_FEEDBACK: 0.0 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 01:00:00 + diff --git a/tests/WE2E/utils.py b/tests/WE2E/utils.py index 33b514d456..d46d52b468 100755 --- a/tests/WE2E/utils.py +++ b/tests/WE2E/utils.py @@ -53,7 +53,7 @@ def print_WE2E_summary(expts_dict: dict, debug: bool = False): expt_details.append('') expt_details.append('-'*REPORT_WIDTH) expt_details.append(f'Detailed summary of experiment {expt}') - expt_details.append(f"in directory {expts_dict[expt]['expt_dir']}") + expt_details.append(f"in directory {os.path.abspath(expts_dict[expt]['expt_dir'])}") expt_details.append(f'{" "*TASK_COLUMN_WIDTH}| Status | Walltime | Core hours used') expt_details.append('-'*REPORT_WIDTH) @@ -123,6 +123,12 @@ def create_expts_dict(expt_dir: str): # Look for FV3LAM_wflow.xml to indicate directories with experiments in them fullpath = os.path.join(expt_dir, item) if not os.path.isdir(fullpath): + # If user is providing an experiment subdir directly, print a warning + if item == "FV3LAM_wflow.xml": + msg = "WARNING: found a rocoto XML in the provided directory!\n" + msg += "This script will only look for experiments in subdirectories\n" + msg += f"of the provided directory {expt_dir}\n" + logging.warning(msg) continue xmlfile = os.path.join(expt_dir, item, 'FV3LAM_wflow.xml') if os.path.isfile(xmlfile): @@ -169,8 +175,8 @@ def calculate_core_hours(expts_dict: dict) -> dict: # Cycle is last 12 characters, task name is rest (minus separating underscore) taskname = task[:-13] # Handle task names that have ensemble and/or fhr info appended with regex - taskname = re.sub('_mem\d{3}', '', taskname) - taskname = re.sub('_f\d{3}', '', taskname) + taskname = re.sub(r'_mem\d{3}', '', taskname) + taskname = re.sub(r'_f\d{3}', '', taskname) nnodes_var = f'NNODES_{taskname.upper()}' if nnodes_var in vdf: nnodes = vdf[nnodes_var] diff --git a/tests/test_python/test_create_diag_table_file.py b/tests/test_python/test_create_diag_table_file.py index f7559536f2..d90dcb3cd0 100644 --- a/tests/test_python/test_create_diag_table_file.py +++ b/tests/test_python/test_create_diag_table_file.py @@ -29,3 +29,4 @@ def setUp(self): set_env_var("DIAG_TABLE_TMPL_FP", diag_table_tmpl_fp) set_env_var("CRES", "C48") set_env_var("CDATE", "2021010106") + set_env_var("UFS_FIRE", False) diff --git a/tests/test_python/test_generate_FV3LAM_wflow.py b/tests/test_python/test_generate_FV3LAM_wflow.py index 48029d21b6..aa4e038f7a 100644 --- a/tests/test_python/test_generate_FV3LAM_wflow.py +++ b/tests/test_python/test_generate_FV3LAM_wflow.py @@ -27,7 +27,7 @@ def test_generate_FV3LAM_wflow(self): # run workflows in separate process to avoid conflict between community and nco settings def run_workflow(USHdir, logfile): - p = Process(target=generate_FV3LAM_wflow, args=(USHdir, logfile)) + p = Process(target=generate_FV3LAM_wflow, args=(USHdir,"config.yaml",logfile)) p.start() p.join() exit_code = p.exitcode @@ -39,6 +39,15 @@ def run_workflow(USHdir, logfile): logfile = "log.generate_FV3LAM_wflow" sed = get_env_var("SED") + # create a build settings file if needed + EXECdir = os.path.join(USHdir, "..", "exec") + build_settings_file = os.path.join(EXECdir, "build_settings.yaml") + if not os.path.exists(build_settings_file): + os.makedirs(EXECdir, exist_ok=True) + with open(build_settings_file, 'w', encoding='utf-8') as build_settings: + build_settings.write('Machine: linux\n') + build_settings.write('Application:\n') + # community test case cp_vrfy(f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") run_command( diff --git a/ufs_srweather_app.settings.in b/ufs_srweather_app.settings.in deleted file mode 100644 index 7958d2706f..0000000000 --- a/ufs_srweather_app.settings.in +++ /dev/null @@ -1,21 +0,0 @@ -# UFS Short Range Weather App Configuration Summary -=================================================== - -# General ---------- -SRWA Version: @SRWA_VERSION@ -Configured On: @CONFIG_DATE@ -Host System: @host_cpu@-@host_vendor@-@host_os@ -Build Directory: @abs_top_builddir@ -Install Prefix: @prefix@ - -# Compiling Options -------------------- -C Compiler: @CC_VERSION@ -CFLAGS: @CFLAGS@ -CPPFLAGS: @CPPFLAGS@ -LDFLAGS: @LDFLAGS@ -Shared Library: @enable_shared@ -Static Library: @enable_static@ -Extra libraries: @LIBS@ - diff --git a/ush/bash_utils/boolify.sh b/ush/bash_utils/boolify.sh index 577b7c16eb..d6857e18e0 100644 --- a/ush/bash_utils/boolify.sh +++ b/ush/bash_utils/boolify.sh @@ -11,15 +11,6 @@ function boolify() { # #----------------------------------------------------------------------- # -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# # Get the name of this function and input. # #----------------------------------------------------------------------- @@ -57,13 +48,4 @@ where: echo "FALSE" fi - # - #----------------------------------------------------------------------- - # - # Restore the shell options saved at the beginning of this script/func- - # tion. - # - #----------------------------------------------------------------------- - # - { restore_shell_opts; } > /dev/null 2>&1 } diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh index 0cfcdc9fdf..3f82dfd1fe 100644 --- a/ush/bash_utils/create_symlink_to_file.sh +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -23,7 +23,6 @@ function create_symlink_to_file() { #----------------------------------------------------------------------- # if [[ $# -lt 2 ]]; then - usage print_err_msg_exit "Function create_symlink_to_file() requires at least two arguments" fi @@ -31,6 +30,9 @@ target=$1 symlink=$2 relative=${3:-TRUE} relative=$(boolify $relative) +if [ "$relative" != "TRUE" ] && [ "$relative" != "FALSE" ]; then + print_err_msg_exit "'relative' must be set to TRUE or FALSE" +fi # #----------------------------------------------------------------------- # @@ -43,17 +45,6 @@ relative=$(boolify $relative) # #----------------------------------------------------------------------- # -# If "relative" is not set (i.e. if it is set to a null string), reset -# it to a default value of "TRUE". Then check that it is set to a vaild -# value. -# -#----------------------------------------------------------------------- -# - valid_vals_relative=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") - check_var_valid_value "relative" "valid_vals_relative" -# -#----------------------------------------------------------------------- -# # Make sure that the target file exists and is a file. # #----------------------------------------------------------------------- diff --git a/ush/config.fire.yaml b/ush/config.fire.yaml new file mode 100644 index 0000000000..fc986f6a6a --- /dev/null +++ b/ush/config.fire.yaml @@ -0,0 +1,67 @@ +metadata: + description: >- + Sample fire config +user: + RUN_ENVIR: community + MACHINE: derecho + ACCOUNT: ACCOUNT_NUMBER +workflow: + USE_CRON_TO_RELAUNCH: true + EXPT_SUBDIR: fire_test_case + CCPP_PHYS_SUITE: FV3_HRRR + PREDEF_GRID_NAME: SUBCONUS_CO_3km + DATE_FIRST_CYCL: '2020081318' + DATE_LAST_CYCL: '2020081318' + FCST_LEN_HRS: 6 # 27 hours of LBCs staged on Derecho + PREEXISTING_DIR_METHOD: rename + VERBOSE: true + COMPILER: intel +task_get_extrn_ics: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_ICS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' + EXTRN_MDL_NAME_ICS: HRRR + FV3GFS_FILE_FMT_ICS: grib2 + EXTRN_MDL_FILES_ICS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' +task_get_extrn_lbcs: + USE_USER_STAGED_EXTRN_FILES: true + EXTRN_MDL_SOURCE_BASEDIR_LBCS: '{{ fire.FIRE_INPUT_DIR }}/HRRR' + EXTRN_MDL_FILES_LBCS: + - '{fyyyymmdd}.hrrr.t{fhh}z.wrfprsf00.grib2' + EXTRN_MDL_NAME_LBCS: HRRR + EXTRN_MDL_LBCS_OFFSET_HRS: 0 + LBC_SPEC_INTVL_HRS: 3 + FV3GFS_FILE_FMT_LBCS: grib2 +task_run_fcst: + QUILTING: true + PRINT_ESMF: True +task_plot_allvars: + COMOUT_REF: "" +fire: + UFS_FIRE: True + FIRE_NUM_TASKS: 1 + FIRE_INPUT_DIR: /glade/derecho/scratch/kavulich/FIRE/2024/test_data + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_ATM_FEEDBACK: 0.0 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 +rocoto: + tasks: + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 01:00:00 + diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 6af52578b8..d298245f14 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -327,7 +327,7 @@ platform: # that will point to a subdirectory (having the name of the grid being # used) under this directory. This variable should be set to a null # string in this file, but it can be specified in the user-specified - # workflow configuration file (EXPT_CONFIG_FN). + # workflow configuration file # #----------------------------------------------------------------------- # @@ -542,9 +542,6 @@ workflow: # # Set file names. Definitions: # - # EXPT_CONFIG_FN: - # Name of the user-specified configuration file for the forecast experiment. - # # CONSTANTS_FN: # Name of the file containing definitions of various mathematical, physical, # and SRW App constants. @@ -618,9 +615,10 @@ workflow: # AQM_RC_TMPL_FN: # Template file name of resource file for NOAA Air Quality Model (AQM) # + # FIRE_NML_FN: + # Name of namelist file for UFS_FIRE capability. #----------------------------------------------------------------------- # - EXPT_CONFIG_FN: "config.yaml" CONSTANTS_FN: "constants.yaml" RGNL_GRID_NML_FN: "regional_grid.nml" @@ -640,6 +638,7 @@ workflow: UFS_CONFIG_FN: "ufs.configure" AQM_RC_FN: "aqm.rc" AQM_RC_TMPL_FN: "aqm.rc" + FIRE_NML_FN: "namelist.fire" # #----------------------------------------------------------------------- @@ -682,6 +681,7 @@ workflow: MODEL_CONFIG_TMPL_FP: '{{ [user.PARMdir, MODEL_CONFIG_FN]|path_join }}' UFS_CONFIG_TMPL_FP: '{{ [user.PARMdir, UFS_CONFIG_FN]|path_join }}' AQM_RC_TMPL_FP: '{{ [user.PARMdir, AQM_RC_TMPL_FN]|path_join }}' + FIRE_NML_BASE_FP: '{{ [user.PARMdir, FIRE_NML_FN]|path_join }}' # #----------------------------------------------------------------------- @@ -756,6 +756,7 @@ workflow: UFS_CONFIG_FP: '{{ [EXPTDIR, UFS_CONFIG_FN]|path_join }}' FV3_NML_FP: '{{ [EXPTDIR, FV3_NML_FN]|path_join }}' FV3_NML_STOCH_FP: '{{ [EXPTDIR, [FV3_NML_FN, "_stoch"]|join ]|path_join }}' + FIRE_NML_FP: '{{ [EXPTDIR, FIRE_NML_FN]|path_join }}' FCST_MODEL: "ufs-weather-model" WFLOW_XML_FN: "FV3LAM_wflow.xml" @@ -923,11 +924,11 @@ workflow: # they are also set to predefined values for the specified grid. # # * If PREDEF_GRID_NAME is set to an empty string, it implies the user - # will provide the native grid parameters in the user-specified - # experiment configuration file (EXPT_CONFIG_FN). In this case, the + # is providing the native grid parameters in the user-specified + # experiment configuration file. In this case, the # grid generation method GRID_GEN_METHOD, the native grid parameters, # the write-component grid parameters, the forecast model's - # main time step DT_ATMOS, and the computational + # main time step DT_ATMOS, and the computational # parameters (LAYOUT_X, LAYOUT_Y, and BLOCKSIZE) must be set in that # configuration file; otherwise, the values of all of these parameters # in this default experiment configuration file will be used. @@ -1629,9 +1630,16 @@ task_make_ics: # FV3-LAM grid. This file will be copied later to a new location and name # changed to fvcom.nc # + # LEVP: + # Number of vertical levels in the atmosphere. In order to change this + # number, the user will additionally need to create a vertical coordinate + # distribution file; this process is described in the Users Guide section + # "Changing the Number of Vertical Levels". This value should be the same + # in both make_ics and make_lbcs. + # # VCOORD_FILE: - # Full path to the file used to set the vertical coordinate in FV3. - # This file should be the same in both make_ics and make_lbcs. + # Full path to the vertical coordinate distribution file. This file should + # be the same in both make_ics and make_lbcs. # #------------------------------------------------------------------------ # @@ -1639,6 +1647,7 @@ task_make_ics: FVCOM_WCSTART: "cold" FVCOM_DIR: "" FVCOM_FILE: "fvcom.nc" + LEVP: 65 VCOORD_FILE: "{{ workflow.FIXam }}/global_hyblev.l65.txt" #---------------------------- @@ -1664,6 +1673,7 @@ task_make_lbcs: KMP_AFFINITY_MAKE_LBCS: "scatter" OMP_NUM_THREADS_MAKE_LBCS: 1 OMP_STACKSIZE_MAKE_LBCS: "1024m" + LEVP: 65 VCOORD_FILE: "{{ workflow.FIXam }}/global_hyblev.l65.txt" #---------------------------- @@ -1784,7 +1794,7 @@ task_run_fcst: # # 1) If the experiment is using a predefined grid, then if the user # sets the parameter in the user-specified experiment configuration - # file (EXPT_CONFIG_FN), that value will be used in the forecast(s). + # file, that value will be used in the forecast(s). # Otherwise, the default value of the parameter for that predefined # grid will be used. # @@ -1918,7 +1928,7 @@ task_run_fcst: QUILTING: true PRINT_ESMF: false - PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X)}}' + PE_MEMBER01: '{{ OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X + WRTCMP_write_groups * WRTCMP_write_tasks_per_group) + fire.FIRE_NUM_TASKS if QUILTING else OMP_NUM_THREADS_RUN_FCST * (LAYOUT_Y * LAYOUT_X) + fire.FIRE_NUM_TASKS }}' WRTCMP_write_groups: "" WRTCMP_write_tasks_per_group: "" @@ -2666,6 +2676,124 @@ cpl_aqm_parm: NEXUS_GFS_SFC_DIR: "" NEXUS_GFS_SFC_ARCHV_DIR: "/NCEPPROD/hpssprod/runhistory" +#---------------------------- +# UFS FIRE config parameters +#----------------------------- +fire: + # + #----------------------------------------------------------------------- + # + # UFS_FIRE: + # Switch for turning on fire simulation + # + # FIRE_INPUT_DIR + # Directory where fire input file (geo_em.d01.nc) can be found + # + # DT_FIRE + # The fire behavior component’s integration timestep + # + # OUTPUT_DT_FIRE + # The fire behavior component’s output timestep + # + # FIRE_NUM_TASKS + # Number of MPI tasks assigned to the FIRE_BEHAVIOR component. + # + + UFS_FIRE: False + FIRE_INPUT_DIR: "" + DT_FIRE: 0.5 + OUTPUT_DT_FIRE: 300 + FIRE_NUM_TASKS: 0 + + + # The following options control namelist values in the Community Fire + # Behavior Model. See the users guide for more information. + # ---------------------------------------------------------------------- + # + # FIRE_WIND_HEIGHT + # Height to interpolate winds to for calculating fire spread rate + # + # FIRE_PRINT_MSG + # Debug print level for the weather model fire core. Levels greater than 1 will print extra + # messages to the log file at run time. + # 0: no extra prints + # 1: Extra prints + # 2: More extra prints + # 3: Even more extra prints + # + # FIRE_ATM_FEEDBACK + # Multiplier for heat fluxes. Use 1.0 for normal two-way coupling. Use 0.0 for one-way coupling. + # Intermediate values will vary the amount of forcing provided from the fire to the dynamical core + # + # FIRE_VISCOSITY + # Artificial viscosity in level set method. Maximum 1, required for FIRE_UPWINDING=0 + # + # FIRE_UPWINDING + # Upwinding scheme used for calculating the normal spread of the fire front + # 0 = Central Difference + # 1 = Standard + # 2 = Godunov + # 3 = ENO1 + # 4 = Sethian + # 5 = 2nd-order Sethian + # 6 = WENO3 + # 7 = WENO5 + # 8 = Hybrid WENO3/ENO1 + # 9 = Hybrid WENO5/ENO1 + # + # FIRE_LSM_ZCOUPLING + # When true, uses FIRE_LSM_ZCOUPLING_REF instead of FIRE_WIND_HEIGHT as a reference height + # to calculate the logarithmic surface layer wind profile + # + # FIRE_LSM_ZCOUPLING_REF + # Reference height from which the velocity at FIRE_WIND_HEIGHT is calculated using a logarithmic profile + # + # FIRE_NUM_IGNITIONS + # Number of fire ignitions. + # + # NOTE: If FIRE_NUM_IGNITIONS > 1, the following variables should be lists with one entry for each ignition + # FIRE_IGNITION_ROS + # + # + # FIRE_IGNITION_START_LAT + # + # + # FIRE_IGNITION_START_LON + # + # + # FIRE_IGNITION_END_LAT + # + # + # FIRE_IGNITION_END_LON + # + # + # FIRE_IGNITION_RADIUS + # + # + # FIRE_IGNITION_START_TIME + # + # + # FIRE_IGNITION_END_TIME + # + + FIRE_WIND_HEIGHT: 5.0 + FIRE_PRINT_MSG: 0 + FIRE_ATM_FEEDBACK: 0.0 + FIRE_VISCOSITY: 0.4 + FIRE_UPWINDING: 9 + FIRE_LSM_ZCOUPLING: False + FIRE_LSM_ZCOUPLING_REF: 60.0 + FIRE_NUM_IGNITIONS: 1 + FIRE_IGNITION_ROS: 0.05 + FIRE_IGNITION_START_LAT: 40.609 + FIRE_IGNITION_START_LON: -105.879 + FIRE_IGNITION_END_LAT: 40.609 + FIRE_IGNITION_END_LON: -105.879 + FIRE_IGNITION_RADIUS: 250 + FIRE_IGNITION_START_TIME: 6480 + FIRE_IGNITION_END_TIME: 7000 + + rocoto: attrs: "" cycledefs: "" diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 466bd3e94b..9ae7530b3a 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -58,7 +58,10 @@ def create_diag_table_file(run_dir): verbose=VERBOSE, ) - settings = {"starttime": CDATE, "cres": CRES} + settings = {"starttime": CDATE, "cres": CRES, "additional_entries": ""} + if UFS_FIRE: + settings["additional_entries"] = \ + '"gfs_phys","fsmoke","fsmoke","fv3_history","all",.false.,"none",2' settings_str = cfg_to_yaml_str(settings) print_info_msg( diff --git a/ush/create_ufs_configure_file.py b/ush/create_ufs_configure_file.py index e97fa25c6a..5bec56fb62 100644 --- a/ush/create_ufs_configure_file.py +++ b/ush/create_ufs_configure_file.py @@ -11,30 +11,66 @@ from uwtools.api.template import render from python_utils import ( - cfg_to_yaml_str, flatten_dict, - import_vars, load_yaml_config, print_info_msg, - print_input_args, ) -def create_ufs_configure_file(run_dir): +def create_ufs_configure_file(run_dir,cfg): """ Creates a UFS configuration file in the specified run directory Args: - run_dir (str): Run directory + run_dir: run directory + cfg: dictionary of config settings Returns: True """ - print_input_args(locals()) + # pylint: disable=undefined-variable - #import all environment variables - import_vars() + # Set necessary variables for each coupled configuration - # pylint: disable=undefined-variable + atm_end = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"]) -1) + aqm_end = str(int(cfg["LAYOUT_X"]) * int(cfg["LAYOUT_Y"]) - 1) + fire_start = str(int(cfg["PE_MEMBER01"]) - int(cfg["FIRE_NUM_TASKS"])) + fire_end = str(int(cfg["PE_MEMBER01"]) - 1) + + atm_petlist_bounds = f'0 {atm_end}' + if cfg["CPL_AQM"]: + earth_component_list = 'ATM AQM' + aqm_petlist_bounds = f'0 {aqm_end}' + atm_omp_num_threads_line = '' + atm_diag_line = '' + runseq = [ f" @{cfg['DT_ATMOS']}\n", + " ATM phase1\n", + " ATM -> AQM\n", + " AQM\n", + " AQM -> ATM\n", + " ATM phase2\n", + " @" ] + elif cfg["UFS_FIRE"]: + earth_component_list = 'ATM FIRE' + atm_omp_num_threads_line = \ + f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" + atm_diag_line = '' + fire_petlist_bounds = f'{fire_start} {fire_end}' + runseq = [ f" @{cfg['DT_ATMOS']}\n", + " ATM -> FIRE\n", + " FIRE -> ATM :remapmethod=conserve\n", + " ATM\n", + " FIRE\n", + " @" ] + else: + earth_component_list = 'ATM' + atm_omp_num_threads_line = \ + f"\nATM_omp_num_threads: {cfg['OMP_NUM_THREADS_RUN_FCST']}" + atm_diag_line = ' Diagnostic = 0' + runseq = [ " ATM" ] + if cfg["PRINT_ESMF"]: + logkindflag = 'ESMF_LOGKIND_MULTI' + else: + logkindflag = 'ESMF_LOGKIND_MULTI_ON_ERROR' # #----------------------------------------------------------------------- # @@ -43,15 +79,13 @@ def create_ufs_configure_file(run_dir): #----------------------------------------------------------------------- # print_info_msg(f''' - Creating a ufs.configure file (\"{UFS_CONFIG_FN}\") in the specified + Creating a ufs.configure file (\"{cfg["UFS_CONFIG_FN"]}\") in the specified run directory (run_dir): - run_dir = \"{run_dir}\"''', verbose=VERBOSE) + {run_dir=}''', verbose=cfg["VERBOSE"]) # # Set output file path # - ufs_config_fp = os.path.join(run_dir, UFS_CONFIG_FN) - pe_member01_m1 = str(int(PE_MEMBER01)-1) - aqm_pe_member01_m1 = str(int(LAYOUT_X*LAYOUT_Y)-1) + ufs_config_fp = os.path.join(run_dir, cfg["UFS_CONFIG_FN"]) # #----------------------------------------------------------------------- # @@ -62,35 +96,43 @@ def create_ufs_configure_file(run_dir): #----------------------------------------------------------------------- # settings = { - "dt_atmos": DT_ATMOS, - "print_esmf": PRINT_ESMF, - "cpl_aqm": CPL_AQM, - "pe_member01_m1": pe_member01_m1, - "aqm_pe_member01_m1": aqm_pe_member01_m1, - "atm_omp_num_threads": OMP_NUM_THREADS_RUN_FCST, + "ufs_fire": cfg["UFS_FIRE"], + "logKindFlag": logkindflag, + "EARTH_cl": earth_component_list, + "ATM_pb": atm_petlist_bounds, + "ATM_omp_num_threads_line": atm_omp_num_threads_line, + "ATM_diag_line": atm_diag_line, + "runseq": runseq, + "AQM_pb": "", + "FIRE_pb": "", + "dt_atmos": cfg["DT_ATMOS"], + "print_esmf": cfg["PRINT_ESMF"], + "cpl_aqm": cfg["CPL_AQM"] } - settings_str = cfg_to_yaml_str(settings) + if cfg["CPL_AQM"]: + settings["AQM_pb"] = aqm_petlist_bounds + if cfg["UFS_FIRE"]: + settings["FIRE_pb"] = fire_petlist_bounds print_info_msg( dedent( f""" - The variable \"settings\" specifying values to be used in the \"{UFS_CONFIG_FN}\" + The variable \"settings\" specifying values to be used in the \"{cfg["UFS_CONFIG_FN"]}\" file has been set as follows:\n - settings =\n\n""" - ) - + settings_str, - verbose=VERBOSE, + {settings=}\n\n""" + ), verbose=cfg["VERBOSE"] ) # #----------------------------------------------------------------------- # - # Call a python script to generate the experiment's actual UFS_CONFIG_FN - # file from the template file. + # Call the uwtools "render" function to fill in jinja expressions contained in the UFS Configure + # file template with the values from the settings variable to create ufs.configure file for this + # experiment # #----------------------------------------------------------------------- # render( - input_file = UFS_CONFIG_TMPL_FP, + input_file = cfg["UFS_CONFIG_TMPL_FP"], output_file = ufs_config_fp, values_src = settings, ) @@ -116,9 +158,6 @@ def _parse_args(argv): if __name__ == "__main__": args = _parse_args(sys.argv[1:]) - cfg = load_yaml_config(args.path_to_defns) - cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) - create_ufs_configure_file( - run_dir=args.run_dir, - ) + conf = load_yaml_config(args.path_to_defns) + confg = flatten_dict(conf) + create_ufs_configure_file(run_dir=args.run_dir,cfg=confg) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 8990f04c6d..2b2f85c8a7 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """ -User interface to create an experiment directory consistent with the user-defined ``config.yaml`` -file. +User interface to create an experiment directory consistent with the user-defined YAML +configuration file. """ # pylint: disable=invalid-name @@ -41,6 +41,7 @@ # pylint: disable=too-many-locals,too-many-branches, too-many-statements def generate_FV3LAM_wflow( ushdir, + config: str = "config.yaml", logfile: str = "log.generate_FV3LAM_wflow", debug: bool = False) -> str: """ @@ -71,7 +72,7 @@ def generate_FV3LAM_wflow( # The setup function reads the user configuration file and fills in # non-user-specified values from config_defaults.yaml - expt_config = setup(ushdir,debug=debug) + expt_config = setup(ushdir,user_config_fn=config,debug=debug) # # ----------------------------------------------------------------------- @@ -93,7 +94,7 @@ def generate_FV3LAM_wflow( # Create a multiline variable that consists of a yaml-compliant string # specifying the values that the jinja variables in the template rocoto # XML should be set to. These values are set either in the user-specified - # workflow configuration file (EXPT_CONFIG_FN) or in the setup() function + # workflow configuration file (config) or in the setup() function # called above. Then call the python script that generates the XML. # # ----------------------------------------------------------------------- @@ -311,6 +312,9 @@ def generate_FV3LAM_wflow( npx = NX + 1 npy = NY + 1 # + # Set npz, which is just LEVP minus 1. + npz = LEVP - 1 + # # For the physics suites that use RUC LSM, set the parameter kice to 9, # Otherwise, leave it unspecified (which means it gets set to the default # value in the forecast model). @@ -379,6 +383,7 @@ def generate_FV3LAM_wflow( "npy": npy, "layout": [LAYOUT_X, LAYOUT_Y], "bc_update_interval": LBC_SPEC_INTVL_HRS, + "npz": npz, }) if CCPP_PHYS_SUITE == "FV3_GFS_v15p2": if CPL_AQM: @@ -442,8 +447,27 @@ def generate_FV3LAM_wflow( "vsvoo1:0.0", "vsvoo2:0.0", "vsvoo3:0.0", "vsvpo1:0.0", "vsvpo2:0.0", "vsvpo3:0.0", "xopn:0.0", "xylmn:0.0", "*:0.2" ] }) + + # If UFS_FIRE, activate appropriate flags and update FIELD_TABLE + if expt_config['fire'].get('UFS_FIRE'): + gfs_physics_nml_dict.update({ + "cpl_fire": True, + }) + field_table_append = """# smoke tracer for UFS_FIRE + "TRACER", "atmos_mod", "fsmoke" + "longname", "fire smoke" + "units", "kg/kg" + "profile_type", "fixed", "surface_value=0.0" /\n""" + + with open(FIELD_TABLE_FP, "a+", encoding='UTF-8') as file: + file.write(field_table_append) + settings["gfs_physics_nml"] = gfs_physics_nml_dict + # Update levp in external_ic_nml; this should be the only variable that needs changing + + settings["external_ic_nml"] = {"levp": LEVP} + # # Add to "settings" the values of those namelist variables that specify # the paths to fixed files in the FIXam directory. As above, these namelist @@ -658,6 +682,64 @@ def generate_FV3LAM_wflow( output_format="nml", update_config=get_nml_config(settings), ) + # + #----------------------------------------------------------------------- + # + # Generate UFS_FIRE namelist if needed. Most variables in the &time section + # will be updated at the run_fcst step + # + #----------------------------------------------------------------------- + # + if expt_config['fire'].get('UFS_FIRE'): + logging.debug("Setting fire namelist values") + fire_nml_dict = {} + fire_nml_dict['atm'] = {} + fire_nml_dict['time'] = {} + fire_nml_dict['fire'] = {} + # Fill in &atm variables + fire_nml_dict['atm']['interval_atm'] = expt_config['task_run_fcst']['DT_ATMOS'] + fire_nml_dict['atm']['kde'] = expt_config['task_make_ics']['LEVP'] + # Fill in &fire and static &time variables + + # These settings must be handled specially below + each_ignit = ["FIRE_IGNITION_ROS", "FIRE_IGNITION_START_LAT", "FIRE_IGNITION_START_LON", + "FIRE_IGNITION_RADIUS", "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME", + "FIRE_IGNITION_END_LAT", "FIRE_IGNITION_END_LON"] + + # These settings do not get added to namelist, or are handled elsewhere + pass_settings = ["UFS_FIRE", "FIRE_INPUT_DIR", "FIRE_NUM_TASKS"] + pass_settings.extend(each_ignit) + + for setting in expt_config['fire']: + if setting in pass_settings: + pass + elif setting == "DT_FIRE": + fire_nml_dict['time']['dt'] = expt_config['fire'][setting] + elif setting == "OUTPUT_DT_FIRE": + fire_nml_dict['time']['interval_output'] = expt_config['fire'][setting] + else: + # For all other settings in config.yaml, convert to lowercase + # and enter into namelist.fire's &fire section + fire_nml_dict['fire'][setting.lower()] = expt_config['fire'][setting] + + # The variables specific to each ignition need special handling: SRW uses a list, but the + # fire model has these settings as separate namelist entries + for i in range(expt_config['fire']['FIRE_NUM_IGNITIONS']): + for setting in each_ignit: + # If not a list, convert to a 1-element list + if not isinstance(expt_config['fire'][setting], list): + expt_config['fire'][setting] = [ expt_config['fire'][setting] ] + + nmle = f"{setting.lower()}{i+1}" + fire_nml_dict['fire'][nmle] = expt_config['fire'][setting][i] + + realize( + input_config=expt_config['workflow']['FIRE_NML_BASE_FP'], + input_format="nml", + output_file=expt_config['workflow']['FIRE_NML_FP'], + output_format="nml", + update_config=get_nml_config(fire_nml_dict), + ) # # ----------------------------------------------------------------------- @@ -668,7 +750,7 @@ def generate_FV3LAM_wflow( # # ----------------------------------------------------------------------- # - cp_vrfy(os.path.join(ushdir, EXPT_CONFIG_FN), EXPTDIR) + cp_vrfy(os.path.join(ushdir, config), EXPTDIR) # # ----------------------------------------------------------------------- @@ -770,6 +852,8 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals description="Script for setting up a forecast and creating a workflow"\ "according to the parameters specified in the config file\n") + parser.add_argument('-c', '--config', default='config.yaml', + help='Name of experiment config file in YAML format') parser.add_argument('-d', '--debug', action='store_true', help='Script will be run in debug mode with more verbose output') pargs = parser.parse_args() @@ -780,7 +864,7 @@ def setup_logging(logfile: str = "log.generate_FV3LAM_wflow", debug: bool = Fals # Call the generate_FV3LAM_wflow function defined above to generate the # experiment/workflow. try: - expt_dir = generate_FV3LAM_wflow(USHdir, wflow_logfile, pargs.debug) + expt_dir = generate_FV3LAM_wflow(USHdir, pargs.config, wflow_logfile, pargs.debug) except: # pylint: disable=bare-except logging.exception( dedent( diff --git a/ush/machine/derecho.yaml b/ush/machine/derecho.yaml index 8bc768732f..d8a3e8f4d4 100644 --- a/ush/machine/derecho.yaml +++ b/ush/machine/derecho.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro + WE2E_TEST_DATA: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index 92d33d7ad2..91e248e57b 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: slurm + WE2E_TEST_DATA: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /gpfs/f5/epic/world-shared/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 189689f30d..1ca55ae270 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm + WE2E_TEST_DATA: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/hercules.yaml b/ush/machine/hercules.yaml index 6a325094da..eddf307091 100644 --- a/ush/machine/hercules.yaml +++ b/ush/machine/hercules.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 80 SCHED: slurm + WE2E_TEST_DATA: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 847530e4eb..b14a4ab9ff 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm + WE2E_TEST_DATA: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /mnt/lfs5/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml index 50de28e751..2b27c0c139 100644 --- a/ush/machine/noaacloud.yaml +++ b/ush/machine/noaacloud.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: '{{ 44 if (user.ACCOUNT == "cz-epic") else 36 if (user.ACCOUNT == "ca-epic") else 28 }}' SCHED: slurm + WE2E_TEST_DATA: /contrib/EPIC/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/odin.yaml b/ush/machine/odin.yaml index 8cd1627da1..6bf51d707b 100644 --- a/ush/machine/odin.yaml +++ b/ush/machine/odin.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm + WE2E_TEST_DATA: /scratch/ywang/UFS_SRW_App/develop DOMAIN_PREGEN_BASEDIR: /scratch/ywang/UFS_SRW_App/develop/FV3LAM_pregen PARTITION_DEFAULT: workq QUEUE_DEFAULT: workq diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 5467160167..285eb34ee2 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm + WE2E_TEST_DATA: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop TEST_CCPA_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc diff --git a/ush/machine/stampede.yaml b/ush/machine/stampede.yaml index 270cd3d593..cb6f1216eb 100644 --- a/ush/machine/stampede.yaml +++ b/ush/machine/stampede.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 68 SCHED: slurm + WE2E_TEST_DATA: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop DOMAIN_PREGEN_BASEDIR: /work2/00315/tg455890/stampede2/UFS_SRW_App/develop/FV3LAM_pregen PARTITION_DEFAULT: normal QUEUE_DEFAULT: normal diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index b8c3625dff..6adb36e6e5 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -2,6 +2,7 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro + WE2E_TEST_DATA: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop TEST_CCPA_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc TEST_MRMS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc TEST_NDAS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc diff --git a/ush/predef_grid_params.yaml b/ush/predef_grid_params.yaml index afcbdddf7c..3dbbcb87a5 100644 --- a/ush/predef_grid_params.yaml +++ b/ush/predef_grid_params.yaml @@ -294,10 +294,77 @@ # #----------------------------------------------------------------------- # -# The RRFS Alaska domain with ~13km cells. +# A 3km grid over Colorado, created for UFS FIRE (but can be used by any experiments) # -# Note: -# This grid has not been thoroughly tested (as of 20201027). +#----------------------------------------------------------------------- +# +"SUBCONUS_CO_3km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -105.6041 + ESGgrid_LAT_CTR: 39.01737 + ESGgrid_DELX: 3000.0 + ESGgrid_DELY: 3000.0 + ESGgrid_NX: 210 + ESGgrid_NY: 200 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 36 + LAYOUT_X: 5 + LAYOUT_Y: 5 + BLOCKSIZE: 40 + QUILTING: + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 5 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -105.6041 + WRTCMP_cen_lat: 39.01736 + WRTCMP_stdlat1: 39.01736 + WRTCMP_stdlat2: 39.01736 + WRTCMP_nx: 207 + WRTCMP_ny: 197 + WRTCMP_lon_lwr_left: -109.0989 + WRTCMP_lat_lwr_left: 36.2794 + WRTCMP_dx: 3000.0 + WRTCMP_dy: 3000.0 +# +#----------------------------------------------------------------------- +# +# A 1km grid over Colorado, created for UFS FIRE (but can be used by any experiments) +# +#----------------------------------------------------------------------- +# +"SUBCONUS_CO_1km": + GRID_GEN_METHOD: "ESGgrid" + ESGgrid_LON_CTR: -105.6041 + ESGgrid_LAT_CTR: 39.01737 + ESGgrid_DELX: 1000.0 + ESGgrid_DELY: 1000.0 + ESGgrid_NX: 630 + ESGgrid_NY: 600 + ESGgrid_PAZI: 0.0 + ESGgrid_WIDE_HALO_WIDTH: 6 + DT_ATMOS: 6 + LAYOUT_X: 9 + LAYOUT_Y: 6 + BLOCKSIZE: 32 + QUILTING: + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: 6 + WRTCMP_output_grid: "lambert_conformal" + WRTCMP_cen_lon: -105.6041 + WRTCMP_cen_lat: 39.01736 + WRTCMP_stdlat1: 39.01736 + WRTCMP_stdlat2: 39.01736 + WRTCMP_nx: 599 + WRTCMP_ny: 570 + WRTCMP_lon_lwr_left: -109.1096 + WRTCMP_lat_lwr_left: 36.2700 + WRTCMP_dx: 1000.0 + WRTCMP_dy: 1000.0 +# +#----------------------------------------------------------------------- +# +# The RRFS Alaska domain with ~13km cells. # #----------------------------------------------------------------------- # @@ -334,9 +401,6 @@ # # The RRFS Alaska domain with ~3km cells. # -# Note: -# This grid has not been thoroughly tested (as of 20201027). -# #----------------------------------------------------------------------- # "RRFS_AK_3km": diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 17440441d5..70d1ef015c 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -221,6 +221,8 @@ def fill_template(template_str, cycle_date, templates_only=False, **kwargs): mem = kwargs.get("mem", "") # ----- + # Set some variables to make the format statement more concise + f_date = cycle_date + dt.timedelta(hours=fcst_hr) cycle_hour = cycle_date.strftime("%H") # One strategy for binning data files at NCEP is to put them into 6 @@ -240,17 +242,26 @@ def fill_template(template_str, cycle_date, templates_only=False, **kwargs): ens_group=ens_group, fcst_hr=fcst_hr, dd=cycle_date.strftime("%d"), + fdd=f_date.strftime("%d"), hh=cycle_hour, + fhh=f_date.strftime("%H"), hh_even=hh_even, jjj=cycle_date.strftime("%j"), + fjjj=f_date.strftime("%j"), mem=mem, min=cycle_date.strftime("%M"), mm=cycle_date.strftime("%m"), + fmm=cycle_date.strftime("%m"), yy=cycle_date.strftime("%y"), + fyy=f_date.strftime("%y"), yyyy=cycle_date.strftime("%Y"), + fyyyy=f_date.strftime("%Y"), yyyymm=cycle_date.strftime("%Y%m"), + fyyyymm=f_date.strftime("%Y%m"), yyyymmdd=cycle_date.strftime("%Y%m%d"), + fyyyymmdd=f_date.strftime("%Y%m%d"), yyyymmddhh=cycle_date.strftime("%Y%m%d%H"), + fyyyymmddhh=f_date.strftime("%Y%m%d%H"), ) if templates_only: diff --git a/ush/setup.py b/ush/setup.py index ab5eac8205..deaebbf7c8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -403,6 +403,18 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False): user_config_fp = os.path.join(USHdir, user_config_fn) expt_config = load_config_for_setup(USHdir, default_config_fp, user_config_fp) + # Load build settings as a dictionary; will be used later to make sure the build is consistent with the user settings + build_config_fp = os.path.join(expt_config["user"].get("EXECdir"), "build_settings.yaml") + build_config = load_config_file(build_config_fp) + logger.debug(f"Read build configuration from {build_config_fp}\n{build_config}") + + # Fail if build machine and config machine are inconsistent + if build_config["Machine"].upper() != expt_config["user"]["MACHINE"]: + logger.critical("ERROR: Machine in build settings file != machine specified in config file") + logger.critical(f"build machine: {build_config['Machine']}") + logger.critical(f"config machine: {expt_config['user']['MACHINE']}") + raise ValueError("Check config settings for correct value for 'machine'") + # Set up some paths relative to the SRW clone expt_config["user"].update(set_srw_paths(USHdir, expt_config)) @@ -690,8 +702,7 @@ def _get_location(xcs, fmt, expt_cfg): ) - # Make sure the vertical coordinate file for both make_lbcs and - # make_ics is the same. + # Make sure the vertical coordinate file and LEVP for both make_lbcs and make_ics is the same. if ics_vcoord := expt_config.get("task_make_ics", {}).get("VCOORD_FILE") != \ (lbcs_vcoord := expt_config.get("task_make_lbcs", {}).get("VCOORD_FILE")): raise ValueError( @@ -707,6 +718,20 @@ def _get_location(xcs, fmt, expt_cfg): VCOORD_FILE: {lbcs_vcoord} """ ) + if ics_levp := expt_config.get("task_make_ics", {}).get("LEVP") != \ + (lbcs_levp := expt_config.get("task_make_lbcs", {}).get("LEVP")): + raise ValueError( + f""" + The number of vertical levels LEVP must be set to the same value for both the + make_ics task and the make_lbcs tasks. They are currently set to: + + make_ics: + LEVP: {ics_levp} + + make_lbcs: + LEVP: {lbcs_levp} + """ + ) # # ----------------------------------------------------------------------- @@ -1467,8 +1492,8 @@ def _dict_find(user_dict, substring): if workflow_config["SDF_USES_THOMPSON_MP"]: - logging.debug(f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP') - logging.debug(f'Setting up links for additional fix files') + logger.debug(f'Selected CCPP suite ({workflow_config["CCPP_PHYS_SUITE"]}) uses Thompson MP') + logger.debug(f'Setting up links for additional fix files') # If the model ICs or BCs are not from RAP or HRRR, they will not contain aerosol # climatology data needed by the Thompson scheme, so we need to provide a separate file @@ -1484,14 +1509,66 @@ def _dict_find(user_dict, substring): for fix_file in fixed_files["THOMPSON_FIX_FILES"]: fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"].append(f"{fix_file} | {fix_file}") - logging.debug(f'New fix file list:\n{fixed_files["FIXgsm_FILES_TO_COPY_TO_FIXam"]=}') - logging.debug(f'New fix file mapping:\n{fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"]=}') + logger.debug(f'New fix file list:\n{fixed_files["FIXgsm_FILES_TO_COPY_TO_FIXam"]=}') + logger.debug(f'New fix file mapping:\n{fixed_files["CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING"]=}') + + # ----------------------------------------------------------------------- + # + # Check that UFS FIRE settings are correct and consistent + # + # ----------------------------------------------------------------------- + fire_conf = expt_config["fire"] + if fire_conf["UFS_FIRE"]: + if build_config["Application"]!="ATMF": + raise Exception("UFS_FIRE == True but UFS SRW has not been built for fire coupling; see users guide for details") + fire_input_file=os.path.join(fire_conf["FIRE_INPUT_DIR"],"geo_em.d01.nc") + if not os.path.isfile(fire_input_file): + raise FileNotFoundError( + dedent( + f""" + The fire input file (geo_em.d01.nc) does not exist in the specified directory: + {fire_conf["FIRE_INPUT_DIR"]} + Check that the specified path is correct, and that the file exists and is readable + """ + ) + ) + if fire_conf["FIRE_NUM_TASKS"] < 1: + raise ValueError("FIRE_NUM_TASKS must be > 0 if UFS_FIRE is True") + elif fire_conf["FIRE_NUM_TASKS"] > 1: + raise ValueError("FIRE_NUM_TASKS > 1 not yet supported") + + if fire_conf["FIRE_NUM_IGNITIONS"] > 5: + raise ValueError(f"Only 5 or fewer fire ignitions supported") + + if fire_conf["FIRE_NUM_IGNITIONS"] > 1: + # These settings all need to be lists for multiple fire ignitions + each_fire = ["FIRE_IGNITION_ROS", "FIRE_IGNITION_START_LAT", "FIRE_IGNITION_START_LON", + "FIRE_IGNITION_END_LAT", "FIRE_IGNITION_END_LON", "FIRE_IGNITION_RADIUS", + "FIRE_IGNITION_START_TIME", "FIRE_IGNITION_END_TIME"] + for setting in each_fire: + if not isinstance(fire_conf[setting], list): + logger.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") + logger.critical(f"{fire_conf[setting]=}") + raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") + if len(fire_conf[setting]) != fire_conf["FIRE_NUM_IGNITIONS"]: + logger.critical(f"{fire_conf['FIRE_NUM_IGNITIONS']=}") + logger.critical(f"{fire_conf[setting]=}") + raise ValueError(f"For FIRE_NUM_IGNITIONS > 1, {setting} must be a list of the same length") + + if fire_conf["FIRE_ATM_FEEDBACK"] > 0.0: + raise ValueError("FIRE_ATM_FEEDBACK > 0 (two-way coupling) not supported in UFS yet") + + if fire_conf["FIRE_UPWINDING"] == 0 and fire_conf["FIRE_VISCOSITY"] == 0.0: + raise ValueError("FIRE_VISCOSITY must be > 0.0 if FIRE_UPWINDING == 0") + else: + if fire_conf["FIRE_NUM_TASKS"] < 1: + logger.warning("UFS_FIRE is not enabled; setting FIRE_NUM_TASKS = 0") # # ----------------------------------------------------------------------- # - # Generate var_defns.sh file in the EXPTDIR. This file contains all + # Generate var_defns.yaml file in the EXPTDIR. This file contains all # the user-specified settings from expt_config. # # ----------------------------------------------------------------------- diff --git a/ush/valid_param_vals.yaml b/ush/valid_param_vals.yaml index fd21b3e1cf..dc3c3b170c 100644 --- a/ush/valid_param_vals.yaml +++ b/ush/valid_param_vals.yaml @@ -24,7 +24,9 @@ valid_vals_PREDEF_GRID_NAME: [ "RRFS_NA_13km", "RRFS_NA_3km", "SUBCONUS_Ind_3km", -"WoFS_3km" +"WoFS_3km", +"SUBCONUS_CO_3km", +"SUBCONUS_CO_1km" ] valid_vals_CCPP_PHYS_SUITE: [ "FV3_GFS_v15p2",