Skip to content

Commit

Permalink
318: test rhist data and local_infile
Browse files Browse the repository at this point in the history 
* small refactor to make "local_infile" configurable
from XML load file.

* fix apparent bugin run_sql  in when load_infile = false

* add test that same result loaded when  local_infile true/false

* add rhist data and test
  • Loading branch information
@John-Sharples
John-Sharples committed Sep 13, 2024
1 parent 823f902 commit 4a4c8b1
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 79 deletions.
9 changes: 9 additions & 0 deletions ...Load/test/data/rhist/ensemble_stat_RRFS_GEFS_GF.SPP.SPPT_RETOP_MRMS_20220507_120000V.stat
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS FULL NEAREST 1 NA NA NA NA ECNT 1888451 10 6.1671 NA 4.10602 -10.99955 12.20766 4.80348 NA NA NA NA NA 11.5788 NA NA
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS FULL NEAREST 1 NA NA NA NA RHIST 1888451 11 4212 9438 10585 10737 12398 16378 22545 34502 58414 131190 1578052
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS FULL NEAREST 1 NA NA NA NA PHIST 321894 0.05 20 4483 5225 3333 3377 5000 4844 5907 6645 9166 12171 15018 18114 22435 23127 26574 27849 29769 35125 34203 29529
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS FULL NEAREST 1 NA NA NA NA RELP 1888451 10 186572.76786 189042.8504 185906.64366 190353.03136 186699.12262 193114.35913 185968.12183 188071.31945 191924.05755 190798.7262
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS CONUS NEAREST 1 NA NA NA NA ECNT 942667 10 5.70001 NA 4.00375 -11.88084 12.68579 4.93033 NA NA NA NA NA 12.38987 NA NA
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS CONUS NEAREST 1 NA NA NA NA RHIST 942667 11 2413 5389 5596 6592 8077 10938 15850 24123 35918 63123 764648
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS CONUS NEAREST 1 NA NA NA NA PHIST 183510 0.05 20 2370 2243 1986 1214 2293 2296 2623 3138 3989 5920 7913 10604 12837 14690 16127 16845 18292 22671 20813 14646
V10.1.1 RRFS_GEFS_GF.SPP.SPPT NA 360000 20220507_120000 20220507_120000 000000 20220507_120039 20220507_120039 RETOP m L0 EchoTop18 km_MSL Z500 MRMS CONUS NEAREST 1 NA NA NA NA RELP 942667 10 93297.89802 95576.19722 92626.90794 93687.40913 93422.46865 96683.925 92821.59802 94516.88452 96700.49008 93333.22143
161 changes: 83 additions & 78 deletions METdbLoad/test/test_met_db_load.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,89 +29,94 @@ def assert_count_rows(cur, table, expected_count):
@pytest.mark.parametrize(
"met_data_dir, met_tool, expected_counts",
[
# (
# POINT_STAT_DATA_DIR,
# "point_stat",
# {
# "line_data_vcnt": 1,
# "line_data_fho": 24,
# "line_data_cts": 24,
# "line_data_ctc": 24,
# "line_data_cnt": 10,
# "line_data_vl1l2": 1,
# },
# ),
# (
# ENSEMBLE_STAT_DATA_DIR,
# "ensemble_stat",
# {
# "line_data_orank": 1426,
# "line_data_phist_bin": 180,
# "line_data_rhist_rank": 84,
# "line_data_phist": 9,
# },
# ),
# (
# GRID_STAT_DATA_DIR,
# "grid_stat",
# {
# "line_data_eclv": 9,
# "line_data_fho": 9,
# "line_data_eclv_pnt": 171,
# "line_data_cts": 9,
# "line_data_ctc": 9,
# "line_data_cnt": 3,
# },
# ),
# (
# MPR_DATA_DIR,
# "grid_stat",
# {
# "line_data_mpr": 1386,
# },
# ),
# (
# TCDIAG_DATA_DIR,
# "tc_diag",
# {
# "line_data_tcmpr": 703,
# "line_data_tcdiag_diag": 592,
# "line_data_tcdiag": 168,
# },
# ),
# (
# MTD_DATA_DIR,
# "mtd",
# {
# "mtd_2d_obj": 278,
# "mtd_3d_obj_single": 8,
# },
# ),
# (
# MODE_DATA_DIR,
# "mtd",
# {
# "mode_cts": 2,
# "mode_obj_pair": 5,
# "mode_obj_single": 6,
# },
# ),
# (
# VSDB_DATA_DIR,
# "vsdb",
# {
# "line_data_ctc": 19,
# "line_data_sl1l2": 5,
# },
# ),
(
RHIST_DATA_DIR,
POINT_STAT_DATA_DIR,
"point_stat",
{
"line_data_vcnt": 1,
"line_data_fho": 24,
"line_data_cts": 24,
"line_data_ctc": 24,
"line_data_cnt": 10,
"line_data_vl1l2": 1,
},
),
(
ENSEMBLE_STAT_DATA_DIR,
"ensemble_stat",
{
{
"line_data_orank": 1426,
"line_data_phist_bin": 180,
"line_data_rhist_rank": 84,
"line_data_phist": 9,
},
),
(
GRID_STAT_DATA_DIR,
"grid_stat",
{
"line_data_eclv": 9,
"line_data_fho": 9,
"line_data_eclv_pnt": 171,
"line_data_cts": 9,
"line_data_ctc": 9,
"line_data_cnt": 3,
},
),
(
MPR_DATA_DIR,
"grid_stat",
{
"line_data_mpr": 1386,
},
),
(
TCDIAG_DATA_DIR,
"tc_diag",
{
"line_data_tcmpr": 703,
"line_data_tcdiag_diag": 592,
"line_data_tcdiag": 168,
},
),
(
MTD_DATA_DIR,
"mtd",
{
"mtd_2d_obj": 278,
"mtd_3d_obj_single": 8,
},
),
(
MODE_DATA_DIR,
"mtd",
{
"mode_cts": 2,
"mode_obj_pair": 5,
"mode_obj_single": 6,
},
),
(
VSDB_DATA_DIR,
"vsdb",
{
"line_data_ctc": 19,
"line_data_sl1l2": 5,
},
),
(
RHIST_DATA_DIR,
"ensemble_stat",
{
"line_data_rhist": 2,
"line_data_rhist_rank":22,
"line_data_phist": 2,
"line_data_phist_bin": 40,
"line_data_ecnt": 2,
"line_data_relp": 2,
"line_data_relp_ens": 20,
},
),
],
)
def test_met_db_table_counts(
Expand Down Expand Up @@ -228,7 +233,7 @@ def test_met_db_indexes(
],
)
def test_local_in_file(emptyDB, testRunSql, tmp_path, met_data_dir, met_tool, expected_counts, local_infile):
# check we get the same result when local_file is on or off
"""check we get the same result when local_file is on or off"""

test_args = dict_to_args(
{
Expand Down
3 changes: 2 additions & 1 deletion METdbLoad/test/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ def abs_path(rel_path):
MPR_DATA_DIR = abs_path("METreformat/test/data/mpr/climo_data")
POINT_STAT_DATA_DIR = abs_path("METreformat/test/data/point_stat")
TCDIAG_DATA_DIR = abs_path("METreformat/test/data/tcdiag_tcmpr")
RHIST_DATA_DIR = abs_path("METreformat/test/data/rhist_phist_relp_orank")


# This data is copied from MET test data
# https://hub.docker.com/r/dtcenter/met-data-output
Expand All @@ -22,6 +22,7 @@ def abs_path(rel_path):

# Very small data sample for testing
VSDB_DATA_DIR = abs_path("METdbLoad/test/data/vsdb/")
RHIST_DATA_DIR = abs_path("METdbLoad/test/data/rhist")

DEFAULT_LOAD_FLAGS = {
"stat_header_db_check": "true",
Expand Down
1 change: 1 addition & 0 deletions METdbLoad/ush/met_db_load.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ def main(args):

if (not file_data.mode_cts_data.empty) or (not file_data.mode_obj_data.empty):
cts_lines = WriteModeSql()

cts_lines.write_mode_data(xml_loadfile.flags,
file_data.mode_cts_data,
file_data.mode_obj_data,
Expand Down

0 comments on commit 4a4c8b1

Please sign in to comment.