Feature 50 improve logging (#333)

* 50: add common logger to met_db_load

* 50: remove nonexistant log attribute 'user'

* 50: add loglevel to test_met_db_load

* 50: use common logger run_sql

* 50: replace logging with common_logger

* 50: Tidy up

* 50: fix circular import

* 50: add 'user' back into log message

METdbLoad/conftest.py

@@ -3,6 +3,7 @@
 import os
 import pymysql
 from pathlib import Path
+from unittest.mock import MagicMock
 
 from METdataio.METdbLoad.ush.run_sql import RunSql
 from METdataio.METdbLoad.test.utils import (
@@ -124,3 +125,8 @@ def load_and_read_xml(
         return XML_LOADFILE
 
     return load_and_read_xml
+
+
+@pytest.fixture
+def mock_logger():
+    return MagicMock()

METdbLoad/test/test_load_specification.xml

@@ -8,7 +8,7 @@
     </connection>
 
     <folder_tmpl>/METdataio/METreformat/test/data/point_stat</folder_tmpl>
-    <verbose>false</verbose>
+    <verbose>true</verbose>
     <insert_size>1</insert_size>
     <stat_header_db_check>true</stat_header_db_check>
     <mode_header_db_check>false</mode_header_db_check>

METdbLoad/test/test_met_db_load.py

@@ -133,6 +133,7 @@ def test_met_db_table_counts(
             "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool)),
             "index": "true",
             "tmpdir": [str(tmp_path)],
+            "loglevel": None,
         }
     )
 
@@ -160,6 +161,7 @@ def test_met_db_indexes(
             ),
             "index": "false",
             "tmpdir": [str(tmp_path)],
+            "loglevel": None
         }
     )
 
@@ -240,6 +242,7 @@ def test_local_in_file(emptyDB, testRunSql, tmp_path, met_data_dir, met_tool, ex
             "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool, local_infile=local_infile)),
             "index": "false",
             "tmpdir": [str(tmp_path)],
+            "loglevel": None,
         }
     )
 

METdbLoad/test/test_run_sql.py

@@ -21,22 +21,23 @@ def populate_some_data(
             ),
             "index": "true",
             "tmpdir": [str(tmp_path)],
+            "loglevel": "DEBUG"
         }
     )
     load_main(test_args)
 
 
-def test_get_file_name(tmp_path, emptyDB, testRunSql):
-    file_name = testRunSql.get_file_name(1, testRunSql.cur)
+def test_get_file_name(tmp_path, emptyDB, testRunSql, mock_logger):
+    file_name = testRunSql.get_file_name(1, testRunSql.cur, mock_logger)
     assert file_name == None
 
     populate_some_data(tmp_path)
 
-    file_name = testRunSql.get_file_name(1, testRunSql.cur)
+    file_name = testRunSql.get_file_name(1, testRunSql.cur, mock_logger)
     assert file_name == "grid_stat_GTG_latlon_060000L_20130827_180000V.stat"
 
     # check for a nonexistant id
-    file_name = testRunSql.get_file_name(9999999, testRunSql.cur)
+    file_name = testRunSql.get_file_name(9999999, testRunSql.cur, mock_logger)
     assert file_name == None
 
 
@@ -47,10 +48,10 @@ def test_get_file_name(tmp_path, emptyDB, testRunSql):
         [False, CN.CREATE_INDEXES_QUERIES],
     ),
 )
-def test_apply_indexes(testRunSql, drop, cmds):
+def test_apply_indexes(testRunSql, drop, cmds, mock_logger):
 
     mock_cursor = mock.MagicMock()
-    testRunSql.apply_indexes(drop, mock_cursor)
+    testRunSql.apply_indexes(drop, mock_cursor, mock_logger)
 
     # Check the correct commands were executed
     assert len(mock_cursor.execute.call_args_list) == len(cmds)
@@ -59,4 +60,4 @@ def test_apply_indexes(testRunSql, drop, cmds):
 
     # check no exception is raised on pymysql error
     mock_cursor.execute.side_effect = OperationalError
-    testRunSql.apply_indexes(drop, mock_cursor)
+    testRunSql.apply_indexes(drop, mock_cursor, mock_logger)

METdbLoad/ush/__init__.py

@@ -0,0 +1 @@
+DEFAULT_LOGLEVEL = "INFO"