diff --git a/METdbLoad/conftest.py b/METdbLoad/conftest.py index be086fa..e5ca635 100644 --- a/METdbLoad/conftest.py +++ b/METdbLoad/conftest.py @@ -8,12 +8,88 @@ top_dir = str(Path(__file__).parents[1]) sys.path.insert(0, os.path.abspath(top_dir)) + +# This is a sample of data copied from test file point_stat_DUP_SINGLE_120000L_20120409_120000V.stat +# found in the METviewer test data. +# TODO: expand this to include other data (e.g. linetypes, met tools, etc.). Probably need to load this from +# disk rather than storing here as a string. +POINT_STAT_DATA = """VERSION MODEL FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_LEV OBS_VAR OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE +V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_120000 20120409_120000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 NA NA NA NA MPR 2 1 001 43.00000 -89.00000 NA 2.00000 275.71640 293.00000 NA NA +V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_120000 20120409_120000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 NA NA NA NA MPR 2 2 002 46.00000 -92.00000 NA 2.00000 272.71640 293.00000 NA NA +V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 >=5.000 >=5.000 NA NA FHO 2 1.00000 1.00000 1.00000 +V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 >=5.000 >=5.000 NA NA CTC 2 2 0 0 0 +V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 >=5.000 >=5.000 NA 0.05000 CTS 2 1.00000 0.34238 1.00000 NA NA 1.00000 0.34238 1.00000 NA NA 1.00000 0.34238 1.00000 NA NA 1.00000 NA NA 1.00000 0.34238 1.00000 NA NA NA NA NA NA NA NA NA NA NA NA 0.00000 0.00000 0.65762 NA NA 1.00000 0.34238 1.00000 NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA NA +V4.2 WRF 120000 20120409_120000 20120409_120000 000000 20120409_103000 20120409_133000 TMP Z2 TMP Z2 ADPSFC FULL UW_MEAN 1 NA NA NA 0.05000 CNT 2 274.21640 255.15709 293.27571 NA NA 2.12132 0.94643 67.69167 NA NA 293.00000 293.00000 293.00000 NA NA 0.00000 0.00000 0.00000 NA NA NA NA NA NA NA NA NA 0 0 0 -18.78360 -37.84291 0.27571 NA NA 2.12132 0.94643 67.69167 NA NA 0.93589 NA NA 18.78360 NA NA 355.07362 NA NA 2.25000 NA NA 18.84340 NA NA -19.98360 NA NA -19.53360 NA NA -18.78360 NA NA -18.03360 NA NA -17.58360 NA NA 1.50000 NA NA 1.50000 NA NA +""" + + +def _populate_xml_load_spec(met_data_dir, + met_tool="point_stat", + host="192.168.0.42"): + """Return the xml load specification with substitute values. + """ + #TODO: determine if other tags require substitution as well + return f""" + + mysql + {host}:3306 + mv_load_test + user + user_pwd + + + {met_data_dir} + true + 1 + true + false + false + false + false + true + false + false + true + true + + + {met_tool} + + + Testing + testing DB load + """ + + +@pytest.fixture +def stat_file_dir(tmp_path): + """Write test stat file and return parent dir.""" + stat_files_dir = tmp_path / "stat_files" + stat_files_dir.mkdir() + + stat_file = stat_files_dir / "point_stat.stat" + with open(stat_file, "w") as text_file: + text_file.write(POINT_STAT_DATA) + return stat_files_dir + + +#TODO: see if we can restrict the scope of this fixture. +@pytest.fixture +def get_xml_test_file(tmp_path, stat_file_dir): + """Write test_load_specification.xml and return path""" + xml_path = tmp_path / "test_load_specification.xml" + with open(xml_path, "w") as text_file: + text_file.write(_populate_xml_load_spec(stat_file_dir)) + return xml_path + + + @pytest.fixture -def get_xml_loadfile(): +def get_xml_loadfile(get_xml_test_file): def load_and_read_xml(): from METdataio.METdbLoad.ush.read_load_xml import XmlLoadFile - XML_FILE = '/Users/venita.hagerty/metviewer/testloadv10fewp3.xml' + XML_FILE = get_xml_test_file XML_LOADFILE = XmlLoadFile(XML_FILE) XML_LOADFILE.read_xml() return XML_LOADFILE diff --git a/METdbLoad/test/test_input.py b/METdbLoad/test/test_input.py index 0f09782..677e458 100644 --- a/METdbLoad/test/test_input.py +++ b/METdbLoad/test/test_input.py @@ -4,7 +4,7 @@ import sys import argparse -def test_argxmlfile(): +def test_argxmlfile(get_xml_test_file): """Check the XML filename.""" sys.argv = ["METdbLoad", "-index", "/Users/venita.hagerty/metviewer/testloads.xml"] diff --git a/METdbLoad/test/test_read_data_files.py b/METdbLoad/test/test_read_data_files.py index d22de2e..869d00b 100644 --- a/METdbLoad/test/test_read_data_files.py +++ b/METdbLoad/test/test_read_data_files.py @@ -8,20 +8,19 @@ def test_counts(get_xml_loadfile): """Count parts of the files loaded in.""" - pytest.skip('Required input file not available') XML_LOADFILE = get_xml_loadfile() # Read all of the data from the data files into a dataframe FILE_DATA = ReadDataFiles() - + # read in the data files, with options specified by XML flags FILE_DATA.read_data(XML_LOADFILE.flags, XML_LOADFILE.load_files, XML_LOADFILE.line_types) # number of files - assert len(XML_LOADFILE.load_files) == 7 + assert len(XML_LOADFILE.load_files) == 1 # number of lines of data - assert FILE_DATA.stat_data.shape[0] == 18106 + assert FILE_DATA.stat_data.shape[0] == 6 # number of line types - assert FILE_DATA.stat_data.line_type.unique().size == 22 + assert FILE_DATA.stat_data.line_type.unique().size == 5 diff --git a/METdbLoad/test/test_tables.py b/METdbLoad/test/test_tables.py index 1e5d8a6..5e06a18 100644 --- a/METdbLoad/test/test_tables.py +++ b/METdbLoad/test/test_tables.py @@ -16,20 +16,7 @@ import sys import math import pymysql - -# *** Connect to a "production/old" database -DB2 = 'mv_ci_prod' - -cnf_file = "/home/runner/work/METdataio/METdataio/headnew/METdbLoad/test/gha.cnf" - -conn2 = pymysql.connect(read_default_file=cnf_file, db=DB2) -cur2 = conn2.cursor() - -# *** Connect to a "test/new" database -DB3 = 'mv_ci_new' - -conn3 = pymysql.connect(read_default_file=cnf_file, db=DB3) -cur3 = conn3.cursor() +import pytest # ****************************************************** @@ -84,518 +71,533 @@ def count_rows(query2, query3): row_num += 1 return row_same +@pytest.mark.skip +def test_something(): + # *** Connect to a "production/old" database + DB2 = 'mv_ci_prod' -# ****************************************************** + cnf_file = "/home/runner/work/METdataio/METdataio/headnew/METdbLoad/test/gha.cnf" -# *** Adjust the number of records tested for each table -QUERY_COUNT = 40 + conn2 = pymysql.connect(read_default_file=cnf_file, db=DB2) + cur2 = conn2.cursor() -# *** Track differences -table_diff = 0 + # *** Connect to a "test/new" database + DB3 = 'mv_ci_new' -# *** Check to see if all full row counts match + conn3 = pymysql.connect(read_default_file=cnf_file, db=DB3) + cur3 = conn3.cursor() + + # ****************************************************** -q_header = 'SELECT count(*) from stat_header' -cur2.execute(q_header) -cur3.execute(q_header) + # *** Adjust the number of records tested for each table + QUERY_COUNT = 40 -print(DB2 + ' has ' + str(cur2.fetchone()[0]) + ' stat_header records and') -print(DB3 + ' has ' + str(cur3.fetchone()[0]) + ' stat_header records\n') + # *** Track differences + table_diff = 0 -# *** Count line_data table rows -q_line2 = "SELECT table_name, table_rows FROM information_schema.tables " + \ - "WHERE table_schema = '" + DB2 + "' AND " + \ - "table_name LIKE 'line_data_%';" + # *** Check to see if all full row counts match -q_line3 = "SELECT table_name, table_rows FROM information_schema.tables " + \ - "WHERE table_schema = '" + DB3 + "' AND " + \ - "table_name LIKE 'line_data_%';" + q_header = 'SELECT count(*) from stat_header' + cur2.execute(q_header) + cur3.execute(q_header) -same = count_rows(q_line2, q_line3) + print(DB2 + ' has ' + str(cur2.fetchone()[0]) + ' stat_header records and') + print(DB3 + ' has ' + str(cur3.fetchone()[0]) + ' stat_header records\n') -if same: - print("%%% No differences in line_data row counts") -else: - table_diff += 1 + # *** Count line_data table rows + q_line2 = "SELECT table_name, table_rows FROM information_schema.tables " + \ + "WHERE table_schema = '" + DB2 + "' AND " + \ + "table_name LIKE 'line_data_%';" -# *** stat_header records -q_header = 'SELECT * from stat_header ' + \ - 'order by model, fcst_var, fcst_lev, vx_mask, fcst_thresh ' + \ - 'limit ' + str(QUERY_COUNT) + ';' + q_line3 = "SELECT table_name, table_rows FROM information_schema.tables " + \ + "WHERE table_schema = '" + DB3 + "' AND " + \ + "table_name LIKE 'line_data_%';" -# show row counts -print("\n*** Row counts for stat_header tables") + same = count_rows(q_line2, q_line3) -result2 = run_query(q_header, cur2) -result3 = run_query(q_header, cur3) + if same: + print("%%% No differences in line_data row counts") + else: + table_diff += 1 -i = 0 -same = True + # *** stat_header records + q_header = 'SELECT * from stat_header ' + \ + 'order by model, fcst_var, fcst_lev, vx_mask, fcst_thresh ' + \ + 'limit ' + str(QUERY_COUNT) + ';' -# compare stat header lines from the two databases -for row in result2: + # show row counts + print("\n*** Row counts for stat_header tables") - if not row[1:] == result3[i][1:]: + result2 = run_query(q_header, cur2) + result3 = run_query(q_header, cur3) - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(row) - print(result3[i]) + i = 0 + same = True - i += 1 + # compare stat header lines from the two databases + for row in result2: -if same: - print("No differences for stat_header") -else: - table_diff += 1 + if not row[1:] == result3[i][1:]: -# *** line data records -line_types = ["cnt", "ctc", "cts", "dmap", "eclv", "ecnt", "enscnt", - "fho", "grad", "isc", "mctc", "mcts", "mpr", "nbrcnt", - "nbrctc", "nbrcts", "orank", "pct", "perc", "phist", - "pjc", "prc", "pstd", "relp", "rhist", "rps", "sl1l2", - "sal1l2", "vl1l2", "val1l2", "seeps", "seeps_mpr", - "ssidx", "ssvar", "vcnt"] + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(row) + print(result3[i]) -for ltype in line_types: + i += 1 - # rows can be created in different orders - match by data file - q_line = 'SELECT * from line_data_' + ltype + \ - ', data_file where ' + \ - 'line_data_' + ltype + \ - '.data_file_id = data_file.data_file_id ' + \ - 'order by filename, line_num limit ' + \ - str(QUERY_COUNT) + ';' + if same: + print("No differences for stat_header") + else: + table_diff += 1 + + # *** line data records + line_types = ["cnt", "ctc", "cts", "dmap", "eclv", "ecnt", "enscnt", + "fho", "grad", "isc", "mctc", "mcts", "mpr", "nbrcnt", + "nbrctc", "nbrcts", "orank", "pct", "perc", "phist", + "pjc", "prc", "pstd", "relp", "rhist", "rps", "sl1l2", + "sal1l2", "vl1l2", "val1l2", "seeps", "seeps_mpr", + "ssidx", "ssvar", "vcnt"] + + for ltype in line_types: + + # rows can be created in different orders - match by data file + q_line = 'SELECT * from line_data_' + ltype + \ + ', data_file where ' + \ + 'line_data_' + ltype + \ + '.data_file_id = data_file.data_file_id ' + \ + 'order by filename, line_num limit ' + \ + str(QUERY_COUNT) + ';' + + # show row counts + print('\n*** Row counts for line_data_' + ltype + ' tables') + + result2 = run_query(q_line, cur2) + result3 = run_query(q_line, cur3) + + if cur2.rowcount > 0 and cur3.rowcount > 0: + + i = 0 + same = True + first = True + + # compare rows from the two databases + for row in result2: + + if different(row[3:-6], row[3:-6]): + same = False + print("*** Different ***") + if first: + for j in range(len(row) - 6): + print(str(row[j]) + " :: " + str(result3[i][j])) + first = False + else: + print(row[3:-6]) + print(" ") + print(result3[i][3:-6]) + + i += 1 + + if same: + print("No differences for line_data_" + ltype) + else: + table_diff += 1 - # show row counts - print('\n*** Row counts for line_data_' + ltype + ' tables') + else: + print("One or both tables are empty") - result2 = run_query(q_line, cur2) - result3 = run_query(q_line, cur3) + # variable length records + vline_types = ["eclv_pnt", "mctc_cnt", "orank_ens", + "pct_thresh", "phist_bin", "pstd_thresh", "pjc_thresh", + "prc_thresh", "relp_ens", "rhist_rank"] - if cur2.rowcount > 0 and cur3.rowcount > 0: + for ltype in vline_types: - i = 0 - same = True - first = True + # get the name of matching line data table + ptype = ltype.split('_')[0] - # compare rows from the two databases - for row in result2: + field_counter = 'i_value' + if ltype == "mctc_cnt": + field_counter = 'i_value, j_value' - if different(row[3:-6], row[3:-6]): - same = False - print("*** Different ***") - if first: - for j in range(len(row) - 6): - print(str(row[j]) + " :: " + str(result3[i][j])) - first = False - else: - print(row[3:-6]) - print(" ") - print(result3[i][3:-6]) + # rows can be created in different orders - match by data file + q_vline = 'SELECT line_data_' + ltype + \ + '.* from line_data_' + ptype + \ + ', line_data_' + ltype + ', data_file WHERE ' + \ + 'line_data_' + ptype + \ + '.data_file_id = data_file.data_file_id AND ' + \ + 'line_data_' + ptype + '.line_data_id = line_data_' + \ + ltype + '.line_data_id ' + \ + ' ORDER BY filename, line_num, line_data_' + ltype + \ + '.line_data_id, ' + field_counter + ' limit ' + \ + str(QUERY_COUNT) + ';' - i += 1 + # show row counts + print('\n*** Row count for line_data_' + ltype + ' tables') - if same: - print("No differences for line_data_" + ltype) - else: - table_diff += 1 + result2 = run_query(q_vline, cur2) + result3 = run_query(q_vline, cur3) - else: - print("One or both tables are empty") - -# variable length records -vline_types = ["eclv_pnt", "mctc_cnt", "orank_ens", - "pct_thresh", "phist_bin", "pstd_thresh", "pjc_thresh", - "prc_thresh", "relp_ens", "rhist_rank"] - -for ltype in vline_types: - - # get the name of matching line data table - ptype = ltype.split('_')[0] - - field_counter = 'i_value' - if ltype == "mctc_cnt": - field_counter = 'i_value, j_value' - - # rows can be created in different orders - match by data file - q_vline = 'SELECT line_data_' + ltype + \ - '.* from line_data_' + ptype + \ - ', line_data_' + ltype + ', data_file WHERE ' + \ - 'line_data_' + ptype + \ - '.data_file_id = data_file.data_file_id AND ' + \ - 'line_data_' + ptype + '.line_data_id = line_data_' + \ - ltype + '.line_data_id ' + \ - ' ORDER BY filename, line_num, line_data_' + ltype + \ - '.line_data_id, ' + field_counter + ' limit ' + \ - str(QUERY_COUNT) + ';' + if cur2.rowcount > 0 and cur3.rowcount > 0: - # show row counts - print('\n*** Row count for line_data_' + ltype + ' tables') + i = 0 + same = True - result2 = run_query(q_vline, cur2) - result3 = run_query(q_vline, cur3) + # compare rows from the two databases + for row in result2: - if cur2.rowcount > 0 and cur3.rowcount > 0: + if different(row[1:], result3[i][1:]): + same = False + print("*** Different ***") + print(row[1:]) + print(result3[i][1:]) - i = 0 - same = True + i += 1 - # compare rows from the two databases - for row in result2: + if same: + print("No differences for line_data_" + ltype) + else: + table_diff += 1 - if different(row[1:], result3[i][1:]): - same = False - print("*** Different ***") - print(row[1:]) - print(result3[i][1:]) + else: + print("One or both tables are empty") - i += 1 + # *** Count mode table rows + q_line2 = "SELECT table_name, table_rows FROM information_schema.tables " + \ + "WHERE table_schema = '" + DB2 + "' AND " + \ + "table_name LIKE 'mode\_%';" - if same: - print("No differences for line_data_" + ltype) - else: - table_diff += 1 + q_line3 = "SELECT table_name, table_rows FROM information_schema.tables " + \ + "WHERE table_schema = '" + DB3 + "' AND " + \ + "table_name LIKE 'mode\_%';" + + same = count_rows(q_line2, q_line3) + if same: + print("\n%%% No differences in mode table row counts") else: - print("One or both tables are empty") + table_diff += 1 -# *** Count mode table rows -q_line2 = "SELECT table_name, table_rows FROM information_schema.tables " + \ - "WHERE table_schema = '" + DB2 + "' AND " + \ - "table_name LIKE 'mode\_%';" + # *** mode_header records + q_header = 'SELECT mode_header.* from mode_header, data_file ' + \ + 'where mode_header.data_file_id = data_file.data_file_id ' + \ + 'order by path, filename, linenumber limit ' + \ + str(QUERY_COUNT) + ';' -q_line3 = "SELECT table_name, table_rows FROM information_schema.tables " + \ - "WHERE table_schema = '" + DB3 + "' AND " + \ - "table_name LIKE 'mode\_%';" + # show row counts + print("\n*** Row counts for mode_header tables") -same = count_rows(q_line2, q_line3) + result2 = run_query(q_header, cur2) + result3 = run_query(q_header, cur3) -if same: - print("\n%%% No differences in mode table row counts") -else: - table_diff += 1 + i = 0 + same = True -# *** mode_header records -q_header = 'SELECT mode_header.* from mode_header, data_file ' + \ - 'where mode_header.data_file_id = data_file.data_file_id ' + \ - 'order by path, filename, linenumber limit ' + \ - str(QUERY_COUNT) + ';' + # compare mode header lines from the two databases + for row in result2: -# show row counts -print("\n*** Row counts for mode_header tables") + rowl = row[3:] + rowr = result3[i][3:] -result2 = run_query(q_header, cur2) -result3 = run_query(q_header, cur3) + if not rowl == rowr: -i = 0 -same = True + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -# compare mode header lines from the two databases -for row in result2: + i += 1 - rowl = row[3:] - rowr = result3[i][3:] + if same: + print("No differences for mode_header") + else: + table_diff += 1 - if not rowl == rowr: + # *** mode_cts records + q_text = 'SELECT * from mode_cts ' + \ + 'order by total, fy_oy, field limit ' + \ + str(QUERY_COUNT) + ';' - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + # show row counts + print("\n*** Row counts for mode_cts tables") - i += 1 + result2 = run_query(q_text, cur2) + result3 = run_query(q_text, cur3) -if same: - print("No differences for mode_header") -else: - table_diff += 1 + i = 0 + same = True -# *** mode_cts records -q_text = 'SELECT * from mode_cts ' + \ - 'order by total, fy_oy, field limit ' + \ - str(QUERY_COUNT) + ';' + # compare mode cts lines from the two databases + for row in result2: -# show row counts -print("\n*** Row counts for mode_cts tables") + rowl = row[1:] + rowr = result3[i][1:] -result2 = run_query(q_text, cur2) -result3 = run_query(q_text, cur3) + if different(rowl, rowr): -i = 0 -same = True + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -# compare mode cts lines from the two databases -for row in result2: + i += 1 - rowl = row[1:] - rowr = result3[i][1:] + if same: + print("No differences for mode_cts") + else: + table_diff += 1 - if different(rowl, rowr): + # *** mode_obj_single records + q_text = 'SELECT * from mode_obj_single ' + \ + 'order by object_id, object_cat, centroid_lat, centroid_lon limit ' + \ + str(QUERY_COUNT) + ';' - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + # show row counts + print("\n*** Row counts for mode_obj_single tables") - i += 1 + result2 = run_query(q_text, cur2) + result3 = run_query(q_text, cur3) -if same: - print("No differences for mode_cts") -else: - table_diff += 1 + i = 0 + same = True -# *** mode_obj_single records -q_text = 'SELECT * from mode_obj_single ' + \ - 'order by object_id, object_cat, centroid_lat, centroid_lon limit ' + \ - str(QUERY_COUNT) + ';' + # compare mode obj single lines from the two databases + for row in result2: -# show row counts -print("\n*** Row counts for mode_obj_single tables") + rowl = row[2:] + rowr = result3[i][2:] -result2 = run_query(q_text, cur2) -result3 = run_query(q_text, cur3) + if different(rowl, rowr): -i = 0 -same = True + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -# compare mode obj single lines from the two databases -for row in result2: + i += 1 - rowl = row[2:] - rowr = result3[i][2:] + if same: + print("\nNo differences for mode_obj_single") + else: + table_diff += 1 + + # *** mode_obj_pair records + q_text = 'SELECT * from mode_obj_pair ' + \ + 'order by mode_header_id, mode_obj_fcst_id, ' + \ + 'mode_obj_obs_id, object_id, object_cat limit ' + \ + str(QUERY_COUNT) + ';' - if different(rowl, rowr): + # show row counts + print("\n*** Row counts for mode_obj_pair tables") - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + result2 = run_query(q_text, cur2) + result3 = run_query(q_text, cur3) - i += 1 + i = 0 + same = True -if same: - print("\nNo differences for mode_obj_single") -else: - table_diff += 1 - -# *** mode_obj_pair records -q_text = 'SELECT * from mode_obj_pair ' + \ - 'order by mode_header_id, mode_obj_fcst_id, ' + \ - 'mode_obj_obs_id, object_id, object_cat limit ' + \ - str(QUERY_COUNT) + ';' + # compare mode obj pair lines from the two databases + for row in result2: -# show row counts -print("\n*** Row counts for mode_obj_pair tables") + rowl = (row[1],) + row[3:] + rowr = (result3[i][1],) + result3[i][3:] -result2 = run_query(q_text, cur2) -result3 = run_query(q_text, cur3) + if different(rowl[3:], rowr[3:]): -i = 0 -same = True + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -# compare mode obj pair lines from the two databases -for row in result2: + i += 1 - rowl = (row[1],) + row[3:] - rowr = (result3[i][1],) + result3[i][3:] + if same: + print("No differences for mode_obj_pair") + else: + table_diff += 1 + + # *** Count mtd table rows + q_line2 = "SELECT table_name, table_rows FROM information_schema.tables " + \ + "WHERE table_schema = '" + DB2 + "' AND " + \ + "table_name LIKE 'mtd\_%';" - if different(rowl[3:], rowr[3:]): + q_line3 = "SELECT table_name, table_rows FROM information_schema.tables " + \ + "WHERE table_schema = '" + DB3 + "' AND " + \ + "table_name LIKE 'mtd\_%';" - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + same = count_rows(q_line2, q_line3) - i += 1 + if same: + print("\n%%% No differences in mtd table row counts") + else: + table_diff += 1 + + # + # *** mtd_header records + # + q_header = 'SELECT * from mtd_header ' + \ + 'order by model, fcst_lead, fcst_valid, fcst_var, revision_id limit ' + \ + str(QUERY_COUNT) + ';' -if same: - print("No differences for mode_obj_pair") -else: - table_diff += 1 - -# *** Count mtd table rows -q_line2 = "SELECT table_name, table_rows FROM information_schema.tables " + \ - "WHERE table_schema = '" + DB2 + "' AND " + \ - "table_name LIKE 'mtd\_%';" + # show row counts + print("\n*** Row counts for mtd_header tables") -q_line3 = "SELECT table_name, table_rows FROM information_schema.tables " + \ - "WHERE table_schema = '" + DB3 + "' AND " + \ - "table_name LIKE 'mtd\_%';" + result2 = run_query(q_header, cur2) + result3 = run_query(q_header, cur3) -same = count_rows(q_line2, q_line3) + i = 0 + same = True -if same: - print("\n%%% No differences in mtd table row counts") -else: - table_diff += 1 - -# -# *** mtd_header records -# -q_header = 'SELECT * from mtd_header ' + \ - 'order by model, fcst_lead, fcst_valid, fcst_var, revision_id limit ' + \ - str(QUERY_COUNT) + ';' + # compare mtd header lines from the two databases + for row in result2: -# show row counts -print("\n*** Row counts for mtd_header tables") + rowl = row[4:] + rowr = result3[i][4:] -result2 = run_query(q_header, cur2) -result3 = run_query(q_header, cur3) + if not rowl == rowr: -i = 0 -same = True + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -# compare mtd header lines from the two databases -for row in result2: + i += 1 - rowl = row[4:] - rowr = result3[i][4:] - - if not rowl == rowr: - - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + if same: + print("No differences for mtd_header") + else: + table_diff += 1 - i += 1 + # + # *** mtd_2d_obj records + # + q_text = 'SELECT mtd_2d_obj.* from mtd_2d_obj, mtd_header, data_file ' + \ + 'where mtd_header.data_file_id = data_file.data_file_id and ' + \ + 'mtd_2d_obj.mtd_header_id = mtd_header.mtd_header_id ' + \ + 'order by path, filename, linenumber, object_id, time_index ' + \ + 'limit ' + str(QUERY_COUNT) + ';' -if same: - print("No differences for mtd_header") -else: - table_diff += 1 + # show row counts + print("\n*** Row counts for mtd_2d_obj tables") -# -# *** mtd_2d_obj records -# -q_text = 'SELECT mtd_2d_obj.* from mtd_2d_obj, mtd_header, data_file ' + \ - 'where mtd_header.data_file_id = data_file.data_file_id and ' + \ - 'mtd_2d_obj.mtd_header_id = mtd_header.mtd_header_id ' + \ - 'order by path, filename, linenumber, object_id, time_index ' + \ - 'limit ' + str(QUERY_COUNT) + ';' + result2 = run_query(q_text, cur2) + result3 = run_query(q_text, cur3) -# show row counts -print("\n*** Row counts for mtd_2d_obj tables") + i = 0 + same = True -result2 = run_query(q_text, cur2) -result3 = run_query(q_text, cur3) + # compare mtd_2d_obj lines from the two databases + for row in result2: + + rowl = row[1:] + rowr = result3[i][1:] + + if different(rowl, rowr): -i = 0 -same = True + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -# compare mtd_2d_obj lines from the two databases -for row in result2: + i += 1 + + if same: + print("No differences for mtd_2d_obj") + else: + table_diff += 1 + + # + # *** mtd_3d_obj_single records + # + q_text = 'SELECT * from mtd_3d_obj_single ' + \ + 'order by object_id, object_cat, centroid_lat, centroid_lon ' + \ + 'limit ' + str(QUERY_COUNT) + ';' - rowl = row[1:] - rowr = result3[i][1:] + # show row counts + print("\n*** Row counts for mtd_3d_obj_single tables") - if different(rowl, rowr): + result2 = run_query(q_text, cur2) + result3 = run_query(q_text, cur3) - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + i = 0 + same = True - i += 1 + # compare mtd_3d_obj_single lines from the two databases + for row in result2: -if same: - print("No differences for mtd_2d_obj") -else: - table_diff += 1 - -# -# *** mtd_3d_obj_single records -# -q_text = 'SELECT * from mtd_3d_obj_single ' + \ - 'order by object_id, object_cat, centroid_lat, centroid_lon ' + \ - 'limit ' + str(QUERY_COUNT) + ';' + rowl = (row[1],) + row[3:] + rowr = (result3[i][1],) + result3[i][3:] -# show row counts -print("\n*** Row counts for mtd_3d_obj_single tables") + if different(rowl[1:], rowr[1:]): -result2 = run_query(q_text, cur2) -result3 = run_query(q_text, cur3) + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) -i = 0 -same = True + i += 1 -# compare mtd_3d_obj_single lines from the two databases -for row in result2: + if same: + print("No differences for mtd_3d_obj_single") + else: + table_diff += 1 - rowl = (row[1],) + row[3:] - rowr = (result3[i][1],) + result3[i][3:] + # + # *** mtd_3d_obj_pair records + # + q_text = 'SELECT * from mtd_3d_obj_pair ' + \ + 'order by object_id, object_cat limit ' + \ + str(QUERY_COUNT) + ';' - if different(rowl[1:], rowr[1:]): + # show row counts + print("\n*** Row counts for mtd_3d_obj_pair tables") - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + result2 = run_query(q_text, cur2) + result3 = run_query(q_text, cur3) - i += 1 + i = 0 + same = True -if same: - print("No differences for mtd_3d_obj_single") -else: - table_diff += 1 + # compare mtd_3d_obj_pair lines from the two databases + for row in result2: -# -# *** mtd_3d_obj_pair records -# -q_text = 'SELECT * from mtd_3d_obj_pair ' + \ - 'order by object_id, object_cat limit ' + \ - str(QUERY_COUNT) + ';' - -# show row counts -print("\n*** Row counts for mtd_3d_obj_pair tables") - -result2 = run_query(q_text, cur2) -result3 = run_query(q_text, cur3) + rowl = (row[1],) + row[3:] + rowr = (result3[i][1],) + result3[i][3:] -i = 0 -same = True - -# compare mtd_3d_obj_pair lines from the two databases -for row in result2: - - rowl = (row[1],) + row[3:] - rowr = (result3[i][1],) + result3[i][3:] + if different(rowl[1:], rowr[1:]): - if different(rowl[1:], rowr[1:]): - - if i > QUERY_COUNT: - continue - same = False - print("*** Different ***") - print(rowl) - print(rowr) + if i > QUERY_COUNT: + continue + same = False + print("*** Different ***") + print(rowl) + print(rowr) - i += 1 + i += 1 -if same: - print("No differences for mtd_3d_obj_pair") -else: - table_diff += 1 + if same: + print("No differences for mtd_3d_obj_pair") + else: + table_diff += 1 -cur2.close -conn2.close() -cur3.close -conn3.close() -sys.exit(table_diff) + cur2.close + conn2.close() + cur3.close + conn3.close() + sys.exit(table_diff) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 0c58f0a..85bf660 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -1,13 +1,8 @@ #!/usr/bin/env python3 """Test reading XML file.""" -import pytest - -REQUIRED_INPUT_MESSAGE = 'Required input file not available' - def test_loadflags(get_xml_loadfile): """Read various flags from XML file.""" - pytest.skip(REQUIRED_INPUT_MESSAGE) XML_LOADFILE = get_xml_loadfile() assert XML_LOADFILE.flags['load_stat'] assert not XML_LOADFILE.flags['load_mode'] @@ -25,23 +20,20 @@ def test_loadflags(get_xml_loadfile): def test_loadgroup(get_xml_loadfile): """Read group and description from XML file.""" - pytest.skip(REQUIRED_INPUT_MESSAGE) XML_LOADFILE = get_xml_loadfile() - assert XML_LOADFILE.group == "vhagerty" - assert XML_LOADFILE.description == "v projects" + assert XML_LOADFILE.group == "Testing" + assert XML_LOADFILE.description == "testing DB load" def test_connection(get_xml_loadfile): """Read connection tags from XML file.""" - pytest.skip(REQUIRED_INPUT_MESSAGE) XML_LOADFILE = get_xml_loadfile() assert XML_LOADFILE.connection['db_host'] == "192.168.0.42" assert XML_LOADFILE.connection['db_port'] == 3306 - assert XML_LOADFILE.connection['db_database'] == "mv_test_3" - assert XML_LOADFILE.connection['db_user'] == "met_admin" + assert XML_LOADFILE.connection['db_database'] == "mv_load_test" + assert XML_LOADFILE.connection['db_user'] == "user" assert XML_LOADFILE.connection['db_management_system'] == "mysql" def test_insertsize(get_xml_loadfile): """Read insert_size from XML file.""" - pytest.skip(REQUIRED_INPUT_MESSAGE) XML_LOADFILE = get_xml_loadfile() assert XML_LOADFILE.insert_size == 1