Feature 213 db load instructions (#214)

* Issue #213 initial commit of script and config files

* support added to create and delete database, and modify xml specification file

* added missing database name to the db drop and create commands

* delete database before creating

* fix syntax on drop and create database, terminating single quote should enclose the db name

* refactored to only handle database prep

* first commit of code that specifically updates the XML specification file

* rename dataclass, remove call to non-existent method

* update logging messages

* fix syntax for granting access

* another fix syntax for granting access

* yet another fix syntax for granting access

* fix grant command by cleaning up spaces and quotations

* fix grant command with terminating quote

* fix schema command

* fix schema command-missing space

* replace check_output with run for subprocess

* use correct path to schema file

* make entries more generic

* Completed instructions, first commit

* fixed syntax for first code block

* Added text to use the existing example for subsetting the data.

* Added explicit directions for running the met_db_load script.

* Fixed formatting of list.

* fix xml_specification setting example

* fix instructions in Load data section to use consistent language

* add troubleshooting content when database is non-existent

* attempt to reformat troubleshooting table

* additional content to troubleshooting using original formatting

* clean up formatting in troubleshooting table

* fix grammar in troubleshooting

* RST syntax errors

METdbLoad/sql/scripts/data_loading_config.yaml

@@ -0,0 +1,32 @@
+
+# Configuration file used to load MET ASCII data into
+# a database.
+
+dbname: dummy_dbname
+username: dbuser
+password: dbpassword
+host: localhost
+port: 1234
+
+# Location (full path and schema file) to the sql schema. Replace with the location
+# of your METdataio source code. PROVIDE THE FULL PATH TO THE SCHEMA FILE, NO RELATIVE PATHS OR
+# ENVIRONMENT VARIABLES.
+schema_location: /full-path-to/mv_mysql.sql
+
+# Name and location of the XML specification file
+xml_specification: /full-path-to/db_load_specification.xml
+
+# Databases are grouped, select an existing group.
+group: Testing
+description: My test database
+
+# Directory (full path) to where the MET data resides.
+data_dir: /path-to-met-data
+
+# Set the appropriate setting to True to indicate what type of data is
+# being loaded.
+load_stat: True
+load_mode: False
+load_mtd: False
+load_mpr: False
+load_orank: False

METdbLoad/sql/scripts/db_load_specification.xml

@@ -0,0 +1,35 @@
+<load_spec>
+  <connection>
+    <management_system>mysql</management_system>
+    <host>localhost:3306</host>
+    <database>mv_integrating_fire</database>
+    <user>mvadmin</user>
+    <password>160GiltVa0D5M</password>
+  </connection>
+
+<folder_tmpl>/scratch/vdunham/</folder_tmpl>
+
+	<verbose>true</verbose>
+	<insert_size>1</insert_size>
+  <stat_header_db_check>true</stat_header_db_check>
+	<mode_header_db_check>false</mode_header_db_check>
+  <mtd_header_db_check>false</mtd_header_db_check>
+	<drop_indexes>false</drop_indexes>
+	<apply_indexes>false</apply_indexes>
+
+	<load_stat>true</load_stat>
+	<load_mode>false</load_mode>
+        <load_mtd>false</load_mtd>
+	<load_mpr>false</load_mpr>
+	<load_orank>false</load_orank>
+
+	<load_val>
+		<field name="met_tool">
+			<val>point_stat</val>
+		</field>
+	</load_val>
+
+	<group>RAL Projects</group>
+	<description>MET output generated for SOARS research.</description>
+
+</load_spec>

METdbLoad/sql/scripts/db_prep.py

@@ -0,0 +1,150 @@
+'''
+    Creates the METviewer database to store MET output. Requires a YAML configuration
+    file (data_loading_config.yaml) with relevant database information (i.e. username,
+    password, etc.).
+'''
+import os.path
+import subprocess
+
+import yaml
+import argparse
+from dataclasses import dataclass
+import logging
+
+logging.basicConfig(encoding='utf-8', level=logging.DEBUG)
+
+
+@dataclass
+class DatabaseInfo:
+    '''
+        Data class for keeping the relevant information for loading the
+        METviewer database.
+    '''
+
+    db_name: str
+    user_name: str
+    password: str
+    host_name: str
+    port_number: int
+    schema_path: str
+    config_file_dir: str
+
+    def __init__(self, config_obj: dict, config_file_dir: str):
+        '''
+
+        Args:
+            config_obj: A dictionary containing the
+                        settings to be used in creating the database.
+        '''
+
+        self.db_name = config_obj['dbname']
+        self.user_name = config_obj['username']
+        self.password = config_obj['password']
+        self.host_name = config_obj['host']
+        self.port_number = config_obj['port']
+        self.schema_path = config_obj['schema_location']
+        self.config_file_dir = config_file_dir
+
+    def create_database(self):
+        '''
+            Create the commands to create the database.
+
+        Returns: None
+
+        '''
+        # Command to create the database, set up permissions, and load the schema.
+        uname_pass_list = ['-u', self.user_name, ' -p', self.password, ' -e ']
+        uname_pass = ''.join(uname_pass_list)
+        create_list = ["'create database ", self.db_name, "'"]
+        create_str = ''.join(create_list)
+        create_cmd = uname_pass + create_str
+        logging.debug(f'database create string: {create_cmd}')
+
+        # Permissions
+        perms_list = ['"',"GRANT INSERT, DELETE, UPDATE, INDEX, DROP ON " ,
+                      self.db_name,
+                     '.* to ', "'mvuser'", "@'%'", '"']
+
+        perms_str = ''.join(perms_list)
+        perms_cmd = uname_pass + perms_str
+        logging.debug(f'database grant permissions string: {perms_cmd}')
+
+
+        # Schema
+        schema_full_path = os.path.join(self.schema_path,
+                                        'METdataio/METdbLoad/sql/mv_mysql.sql')
+        schema_list = [ "-umvadmin -p",self.password, " ", self.db_name, ' < ',
+                        schema_full_path]
+        schema_cmd = ''.join(schema_list)
+        logging.debug(f'Schema command: {schema_cmd}')
+
+
+
+        try:
+            self.delete_database()
+        except subprocess.CalledProcessError:
+            logging.info("Database doesn't exist. Ignoring this error.")
+            pass
+
+        try:
+            create_db = subprocess.run(['mysql', create_cmd])
+            db_permissions = subprocess.run(['mysql', perms_cmd])
+            db_schema = subprocess.run(['mysql', schema_cmd])
+        except subprocess.CalledProcessError:
+            logging.error('Error in executing mysql commands')
+
+    def delete_database(self):
+        '''
+           Create the commands to delete a database.
+        Returns: None
+
+        '''
+
+        # Command to delete the database
+        uname_pass_list = ['-u', self.user_name, ' -p', self.password, ' -e ']
+        uname_pass = ''.join(uname_pass_list)
+        drop_list = ["'drop database ", self.db_name, "'"]
+        drop_str = ''.join(drop_list)
+        drop_cmd = uname_pass + drop_str
+        logging.debug(f'Drop database command: {drop_cmd}')
+
+        try:
+            _ = subprocess.run(['mysql', drop_cmd])
+
+        except subprocess.CalledProcessError:
+            logging.error('Error in executing mysql commands')
+
+
+if __name__ == "__main__":
+
+    # Create a parser
+    parser = argparse.ArgumentParser()
+
+    # Add arguments to the parser
+    parser.add_argument('action')
+    parser.add_argument('config_file')
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Get arguments value
+    action = args.action
+    config_file = args.config_file
+
+    action_requested = str(action).lower()
+    logging.debug(f'Action requested: {action_requested}')
+    logging.debug(f'YAML Config file to use: {str(config_file)}')
+    config_file_dir = os.path.dirname(config_file)
+    logging.debug(f'Directory of config file: {config_file_dir}')
+
+    with open(config_file, 'r') as cf:
+        db_config_info = yaml.safe_load(cf)
+        db_loader = DatabaseInfo(db_config_info, config_file_dir)
+        if action_requested == 'create':
+            db_loader.create_database()
+        elif action_requested == 'delete':
+            db_loader.delete_database()
+        else:
+            logging.warning(
+                f'{action_requested} is not a supported option. Only "create" and '
+                f'"delete" are supported options.')

METdbLoad/sql/scripts/generate_xml_spec.py

@@ -0,0 +1,121 @@
+'''
+    Creates the METviewer database to store MET output.
+'''
+import os.path
+import subprocess
+
+import yaml
+import argparse
+from dataclasses import dataclass
+import logging
+
+logging.basicConfig(encoding='utf-8', level=logging.DEBUG)
+
+
+@dataclass
+class DatabaseLoadingInfo:
+    '''
+        Data class for keeping the relevant information for loading the
+        METviewer database.
+    '''
+
+    db_name: str
+    user_name: str
+    password: str
+    host_name: str
+    port_number: int
+    group: str
+    schema_path: str
+    data_dir: str
+    xml_spec_file: str
+    load_stat: bool
+    load_mode: bool
+    load_mtd: bool
+    load_mpr: bool
+    load_orank: bool
+    config_file_dir: str
+
+    def __init__(self, config_obj: dict, config_file_dir:str):
+        '''
+
+        Args:
+            config_obj: A dictionary containing the
+                        settings to be used in creating the database.
+        '''
+
+        self.db_name = config_obj['dbname']
+        self.user_name = config_obj['username']
+        self.password = config_obj['password']
+        self.host_name = config_obj['host']
+        self.port_number = config_obj['port']
+        self.group = config_obj['group']
+        self.schema_path = config_obj['schema_location']
+        self.data_dir = config_obj['data_dir']
+        self.xml_spec_file = config_obj['xml_specification']
+        self.load_stat = config_obj['load_stat']
+        self.load_mode = config_obj['load_mode']
+        self.load_mtd = config_obj['load_mtd']
+        self.load_mpr = config_obj['load_mpr']
+        self.load_orank = config_obj['load_orank']
+        self.description = config_obj['description']
+        self.config_file_dir = config_file_dir
+
+
+    def update_spec_file(self):
+        '''
+           Edit the XML specification file to reflect the settings in the
+           YAML configuration file.
+        '''
+
+        # Assign the host with the host and port assigned in the YAML config file
+        import xml.etree.ElementTree as et
+        tree = et.parse(self.xml_spec_file)
+        root = tree.getroot()
+
+        for host in root.iter('host'):
+            host.text = self.host_name + ":" + str(self.port_number)
+
+        for dbname in root.iter('database'):
+            dbname.text = self.db_name
+
+        for user in root.iter('user'):
+            user.text = self.user_name
+
+        for password in root.iter('password'):
+            password.text = self.password
+
+        for data_folder in root.iter('folder_tmpl'):
+            data_folder.text = self.data_dir
+
+        for group in root.iter('group'):
+            group.text = self.group
+
+        for desc in root.iter('description'):
+            desc.text = self.description
+
+        tree.write(os.path.join(self.config_file_dir, 'load_met.xml'))
+
+
+
+if __name__ == "__main__":
+
+    # Create a parser
+    parser = argparse.ArgumentParser()
+
+    # Add arguments to the parser
+    parser.add_argument('config_file')
+
+    # Parse the arguments
+    args = parser.parse_args()
+
+    # Get arguments value
+    config_file = args.config_file
+
+    logging.debug(f'Config file to use: {str(config_file)}')
+    config_file_dir = os.path.dirname(config_file)
+    logging.debug(f'Directory of config file: {config_file_dir}')
+
+    with open(config_file, 'r') as cf:
+        db_config_info = yaml.safe_load(cf)
+        db_loader = DatabaseLoadingInfo(db_config_info, config_file_dir)
+        db_loader.update_spec_file()