Skip to content

Commit

Permalink
318: Add test and test data for met_db_load (#328)
Browse files Browse the repository at this point in the history 
* Add new test data dir and sample data for tests

* Adds test/utils.py for test helper functions

* Expands test_met_db_load to count loaded rows from test data

* Removes test/test_input.py as it didn't test anything

* Tidy up whitespace, string quotes  and reformatting conftest
  • Loading branch information
@John-Sharples
John-Sharples committed Sep 6, 2024
1 parent bb65007 commit 61842c0
Show file tree
Hide file tree
Showing 9 changed files with 533 additions and 158 deletions.
129 changes: 33 additions & 96 deletions METdbLoad/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
import sys
import os
import pymysql
import logging
from pathlib import Path
from unittest.mock import patch

from METdataio.METdbLoad.ush.read_data_files import ReadDataFiles
from METdataio.METdbLoad.ush.run_sql import RunSql
from METdataio.METdbLoad.test.utils import (
get_xml_test_file,
POINT_STAT_DATA_DIR,
)


# add METdataio directory to path so packages can be found
Expand All @@ -16,26 +18,26 @@

def parse_sql(filename):
"""Parse a .sql file and return a list of SQL statements"""
data = open(filename, 'r').readlines()
data = open(filename, "r").readlines()
stmts = []
DELIMITER = ';'
stmt = ''
DELIMITER = ";"
stmt = ""

for line in data:
if not line.strip():
continue

if line.startswith('--'):
if line.startswith("--"):
continue

if (DELIMITER not in line):
if DELIMITER not in line:
stmt += line
continue

if stmt:
stmt += line
stmts.append(stmt.strip())
stmt = ''
stmt = ""
else:
stmts.append(line.strip())
return stmts
Expand All @@ -47,10 +49,10 @@ def maria_conn():
"""
try:
conn = pymysql.connect(
host='localhost',
host="localhost",
port=3306,
user='root',
password='root_password',
user="root",
password="root_password",
)

except Exception as e:
Expand All @@ -76,15 +78,15 @@ def emptyDB():
conn.close()

db_conn = pymysql.connect(
host='localhost',
port=3306,
user='root',
password='root_password',
database='mv_test',
autocommit=True,
)
host="localhost",
port=3306,
user="root",
password="root_password",
database="mv_test",
autocommit=True,
)

sql_statements = parse_sql(Path(TOP_DIR) / 'METdbLoad/sql/mv_mysql.sql')
sql_statements = parse_sql(Path(TOP_DIR) / "METdbLoad/sql/mv_mysql.sql")

with db_conn.cursor() as cur:
for stm in sql_statements:
Expand All @@ -95,98 +97,33 @@ def emptyDB():

@pytest.fixture
def testRunSql():
"""Return an instance of RunSql with a connection.
"""
"""Return an instance of RunSql with a connection."""
connection = {
'db_host': 'localhost',
'db_port': 3306,
'db_user': 'root',
'db_password': 'root_password',
'db_database': 'mv_test',
"db_host": "localhost",
"db_port": 3306,
"db_user": "root",
"db_password": "root_password",
"db_database": "mv_test",
}

testRunSql = RunSql()
testRunSql.sql_on(connection)
return testRunSql


# This is a sample of data copied from test file point_stat_DUP_SINGLE_120000L_20120409_120000V.stat
# found in the METviewer test data.
# TODO: expand this to include other data (e.g. linetypes, met tools, etc.). Probably need to load this from
# disk rather than storing here as a string.
POINT_STAT_DATA = """VERSION MODEL FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_LEV OBS_VAR OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE
V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_120000 20120409_120000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 NA NA NA NA MPR 2 1 001 43.00000 -89.00000 NA 2.00000 275.71640 293.00000 NA NA
V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_120000 20120409_120000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 NA NA NA NA MPR 2 2 002 46.00000 -92.00000 NA 2.00000 272.71640 293.00000 NA NA
V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 >=5.000 >=5.000 NA NA FHO 2 1.00000 1.00000 1.00000
V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 >=5.000 >=5.000 NA NA CTC 2 2 0 0 0
V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 >=5.000 >=5.000 NA 0.05000 CTS 2 1.00000 0.34238 1.00000 NA NA 1.00000 0.34238 1.00000 NA NA 1.00000 0.34238 1.00000 NA NA 1.00000 NA NA 1.00000 0.34238 1.00000 NA NA NA NA NA NA NA NA NA NA NA NA 0.00000 0.00000 0.65762 NA NA 1.00000 0.34238 1.00000 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 NA NA NA 0.05000 CNT 2 274.21640 255.15709 293.27571 NA NA 2.12132 0.94643 67.69167 NA NA 293.00000 293.00000 293.00000 NA NA 0.00000 0.00000 0.00000 NA NA NA NA NA NA NA NA NA 0 0 0 -18.78360 -37.84291 0.27571 NA NA 2.12132 0.94643 67.69167 NA NA 0.93589 NA NA 18.78360 NA NA 355.07362 NA NA 2.25000 NA NA 18.84340 NA NA -19.98360 NA NA -19.53360 NA NA -18.78360 NA NA -18.03360 NA NA -17.58360 NA NA 1.50000 NA NA 1.50000 NA NA
"""


def _populate_xml_load_spec(met_data_dir,
met_tool="point_stat",
host="localhost"):
"""Return the xml load specification with substitute values.
"""
#TODO: determine if other tags require substitution as well
return f"""<load_spec>
<connection>
<management_system>mysql</management_system>
<host>{host}:3306</host>
<database>mv_test</database>
<user>root</user>
<password>root_password</password>
</connection>
<folder_tmpl>{met_data_dir}</folder_tmpl>
<verbose>true</verbose>
<insert_size>1</insert_size>
<stat_header_db_check>true</stat_header_db_check>
<mode_header_db_check>false</mode_header_db_check>
<mtd_header_db_check>false</mtd_header_db_check>
<drop_indexes>false</drop_indexes>
<apply_indexes>false</apply_indexes>
<load_stat>true</load_stat>
<load_mode>true</load_mode>
<load_mtd>true</load_mtd>
<load_mpr>true</load_mpr>
<load_orank>true</load_orank>
<force_dup_file>true</force_dup_file>
<load_val>
<field name="met_tool">
<val>{met_tool}</val>
</field>
</load_val>
<group>Testing</group>
<description>testing DB load</description>
</load_spec>"""


# TODO: give access to the other test data
@pytest.fixture
def point_stat_file_dir(tmp_path):
"""Write test stat file and return parent dir."""
return str(Path(TOP_DIR) / 'METreformat/test/data/point_stat' )


#TODO: see if we can restrict the scope of this fixture.
@pytest.fixture
def get_xml_test_file(tmp_path, point_stat_file_dir):
"""Write test_load_specification.xml and return path"""
xml_path = tmp_path / "test_load_specification.xml"
with open(xml_path, "w") as text_file:
text_file.write(_populate_xml_load_spec(point_stat_file_dir))
return xml_path

def point_stat_xml_file(tmp_path):
"""Get xml load file for point_stat test data."""
data_path = Path(TOP_DIR) / POINT_STAT_DATA_DIR
return get_xml_test_file(tmp_path, data_path, "point_stat")


@pytest.fixture
def get_xml_loadfile(get_xml_test_file):
def get_xml_loadfile(point_stat_xml_file):
def load_and_read_xml():
from METdataio.METdbLoad.ush.read_load_xml import XmlLoadFile

XML_FILE = get_xml_test_file
XML_FILE = point_stat_xml_file
XML_LOADFILE = XmlLoadFile(XML_FILE)
XML_LOADFILE.read_xml()
return XML_LOADFILE
Expand Down
3 changes: 3 additions & 0 deletions METdbLoad/test/data/mode/mode_MASK_POLY_300000L_20120410_180000V_060000A_cts.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE FIELD TOTAL FY_OY FY_ON FN_OY FN_ON BASER FMEAN ACC FBIAS PODY PODN POFD FAR CSI GSS HK HSS ODDS LODDS ORSS EDS SEDS EDI SEDI BAGSS
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 RAW 2461 228 488 50 1695 0.11296 0.29094 0.78139 2.57554 0.82014 0.77645 0.22355 0.68156 0.29765 0.21474 0.5966 0.35355 15.83852 2.76245 0.88122 0.83331 0.43564 0.76624 0.75373 0.21605
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 OBJECT 2461 235 498 52 1676 0.11662 0.29785 0.77651 2.55401 0.81882 0.77093 0.22907 0.6794 0.29936 0.21374 0.58974 0.35221 15.2093 2.72191 0.87661 0.82978 0.43056 0.76112 0.74736 0.21532
12 changes: 12 additions & 0 deletions METdbLoad/test/data/mode/mode_MASK_POLY_300000L_20120410_180000V_060000A_obj.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
VERSION MODEL N_VALID GRID_RES DESC FCST_LEAD FCST_VALID FCST_ACCUM OBS_LEAD OBS_VALID OBS_ACCUM FCST_RAD FCST_THR OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE OBJECT_ID OBJECT_CAT CENTROID_X CENTROID_Y CENTROID_LAT CENTROID_LON AXIS_ANG LENGTH WIDTH AREA AREA_THRESH CURVATURE CURVATURE_X CURVATURE_Y COMPLEXITY INTENSITY_10 INTENSITY_25 INTENSITY_50 INTENSITY_75 INTENSITY_90 INTENSITY_50 INTENSITY_SUM CENTROID_DIST BOUNDARY_DIST CONVEX_HULL_DIST ANGLE_DIFF ASPECT_DIFF AREA_RATIO INTERSECTION_AREA UNION_AREA SYMMETRIC_DIFF INTERSECTION_OVER_AREA CURVATURE_RATIO COMPLEXITY_RATIO PERCENTILE_INTENSITY_RATIO INTEREST
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 F001 CF001 80.36074 96.51534 39.61744 -122.96297 -69.71491 56.85092 20.18892 730 687 145.02061 97.27916 165.44052 0.14269 5.569 7.3725 9.44 11.91 15.544 9.44 7295.71999 NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 F002 CF001 75.125 120.875 41.60843 -123.6629 45 2.82843 2.12132 3 3 159.38557 120.44762 176.49756 0.14286 5.996 6.035 6.1 6.2 6.26 6.1 18.37 NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 O001 CO001 84.38693 104.56281 40.29206 -122.56533 87.94059 24.09231 11.02883 161 141 162.04809 79.97732 194.76183 0.19701 4.87 5.84 7.06 8.75 11.23 7.06 1239.99 NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 O002 CO001 85.9172 83.36306 38.56085 -122.32682 87.18156 16.07899 12.32968 126 115 135.19602 116.59531 137.93944 0.16279 5.365 6.4525 7.845 9.7825 12.405 7.845 1058.84 NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 F001_O001 CF001_CO001 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA 8.99845 0 0 22.3445 0.10265 4.53416 112 779 667 0.69565 0.89492 0.72428 0.74788 0.90889
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 F001_O002 CF001_CO001 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA 14.27784 0 0 23.10352 0.4117 5.79365 123 733 610 0.97619 0.93225 0.87652 0.83104 0.89471
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 F002_O001 CF001_CO001 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA 18.75822 6.40312 6.40312 42.94059 0.29223 0.018634 0 164 164 0 0.98357 0.72514 0.86402 0.51327
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 F002_O002 CF001_CO001 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA 39.03354 29.41088 29.41088 42.18156 0.01682 0.02381 0 129 129 0 0.84823 0.87755 0.77757 0.22928
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 CF001 CF001 80.30984 96.75213 39.63667 -122.96947 -69.98662 56.81848 20.01174 733 690 145.31439 96.78289 166.10212 0.14519 5.582 7.35 9.43 11.91 15.526 9.43 7314.08999 NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 CO001 CO001 85.0618 95.21348 39.52563 -122.45784 -86.97497 41.31227 12.19437 287 256 154.53498 85.38557 181.40353 0.3134 4.916 6.06 7.42 9.44 11.736 7.42 2298.83 NA NA NA NA NA NA NA NA NA NA NA NA NA NA
V12.0.0 FCST 2461 9 NA 300000 20120410_180000 060000 060000 20120410_180000 060000 2 >=5.0 2 >=5.0 APCP_06 kg/m^2 A6 APCP_06 kg/m^2 A06 STAGE4 CF001_CO001 CF001_CO001 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA 4.99485 0 0 16.98835 0.057029 2.55401 235 785 550 0.81882 0.94033 0.46328 0.78685 0.94028
Loading

0 comments on commit 61842c0

Please sign in to comment.